from pyspark import SparkContext, SparkConf, SQLContext
from pyspark.sql import functions as F
import pandas as pd
from pyspark.sql.types import IntegerType
import numpy as np
sc = SparkContext("local", "bike")
sqlContext = SQLContext(sc)
df = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load('201904-citibike-tripdata.csv')
df5 = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load('201905-citibike-tripdata.csv')
df6 = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load('201906-citibike-tripdata.csv')
df7 = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load('201907-citibike-tripdata.csv')
df=df.union(df5)
df=df.union(df6)
df=df.union(df7)
df = df.filter(df['start station id']==519)
hour_window = F.window(F.col("starttime"), "1 hour", "1 hour").start.alias("starttime")
hour_num = df.groupBy(hour_window).count()
print(hour_num.count())
hour_num.show()
2563 +-------------------+-----+ | starttime|count| +-------------------+-----+ |2019-04-02 23:00:00| 5| |2019-04-13 19:00:00| 8| |2019-04-14 12:00:00| 16| |2019-04-22 22:00:00| 11| |2019-05-02 04:00:00| 1| |2019-05-31 10:00:00| 25| |2019-06-08 01:00:00| 1| |2019-06-17 20:00:00| 12| |2019-07-03 17:00:00| 93| |2019-04-19 16:00:00| 38| |2019-05-02 21:00:00| 10| |2019-05-04 04:00:00| 1| |2019-05-31 22:00:00| 5| |2019-05-31 23:00:00| 8| |2019-06-16 07:00:00| 1| |2019-06-22 19:00:00| 4| |2019-06-24 18:00:00| 105| |2019-07-15 08:00:00| 92| |2019-07-18 10:00:00| 7| |2019-04-03 04:00:00| 1| +-------------------+-----+ only showing top 20 rows
df_weather = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load('processed_weather.csv')
print(df_weather.count())
df_weather.show()
2924 +-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+ | starttime| SPD| GUS|CLR|SCT|BKN|OVC|OBS|POB| VSB| TEMP| DEWP| SLP| ALT| STP| PCP01| +-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+ |2019-04-01 00:00:00| 0.4111950573077732| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.202718242869458|-2.4059857964589435|-0.00771000909058...|-0.03138250547365...|-0.02748200443494122|-0.17811912685987313| |2019-04-01 01:00:00| 2.8743640151934144| 3.3341854908713513| 0| 1| 0| 0| 0| 0|0.4638614904960641| -2.367325382057515|-2.5515963785584366| 0.10097350748358193| 0.07409157039460014| 0.06639344062031857|-0.17811912685987313| |2019-04-01 02:00:00| 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.531932521245572|-2.6244016696081833| 0.1630783740973855| 0.12682860832873663| 0.1289770706571525|-0.17811912685987313| |2019-04-01 03:00:00| 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6244016696081833| 0.2562356740180996| 0.23230268419699088| 0.2384984232216252|-0.17811912685987313| |2019-04-01 04:00:00| 1.6427795362505941|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296| 0.4270240572060726| 0.3905137979993629| 0.39495749831371896|-0.17811912685987313| |2019-04-01 05:00:00| 1.3348834165148888| 3.0033915720566178| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.696539660433629|-2.6972069606579296| 0.5822862237405904| 0.5487249118017536| 0.5514165734058126|-0.17811912685987313| |2019-04-01 06:00:00| 1.3348834165148888| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.7788432300276575| -2.770012251707676| 0.7220221736216527| 0.7069360256041444| 0.7078756484979064|-0.17811912685987313| |2019-04-01 07:00:00| 1.9506756559862992| 3.3341854908713513| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.7788432300276575| -2.770012251707676| 0.9238629901165186| 0.9178841773406529| 0.9112724461176211|-0.17811912685987313| |2019-04-01 08:00:00| 1.3348834165148888| 2.5072006938345175| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296| 1.0014940733837774| 0.9706212152747894| 0.973856076154455|-0.17811912685987313| |2019-04-01 09:00:00| 1.3348834165148888|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.4496289516515435|-2.6972069606579296| 1.1257038066114022| 1.0760952911430437| 1.0833774287189277|-0.17811912685987313| |2019-04-01 10:00:00| 1.0269872967791835| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.2850218124634862|-2.6972069606579296| 1.2499135398390093| 1.2343064049454158| 1.2398365038110215|-0.17811912685987313| |2019-04-01 11:00:00| 1.3348834165148888| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.202718242869458|-2.6972069606579296| 1.3120184064528129| 1.287043442879552| 1.2867742263386424|-0.17811912685987313| |2019-04-01 12:00:00| 2.2585717757220043| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.038111103681401|-2.8428175427574227| 1.3585970564131788| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| |2019-04-01 13:00:00|-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.8735039644933438| -2.915622833807169| 1.3585970564131788| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| |2019-04-01 14:00:00|-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-2.9884281248569153| 1.3741232730666164| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| |2019-04-01 15:00:00| 1.9506756559862992| 3.0033915720566178| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.1340387069564084| 1.3896494897200717| 1.3925175187478065| 1.3962955789031153|-0.17811912685987313| |2019-04-01 16:00:00| 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.2068439980061547| 1.4362281396804375| 1.4452545566819428| 1.4432333014307361|-0.17811912685987313| |2019-04-01 17:00:00| 2.2585717757220043| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.2796492890559015| 1.5914903062149552| 1.5507286325501972| 1.5527546539952088|-0.17811912685987313| |2019-04-01 18:00:00| 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.8735039644933438|-3.2068439980061547| 1.7157000394425623| 1.708939746352569| 1.7092137290873026|-0.17811912685987313| |2019-04-01 19:00:00| 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.9558075340873724|-3.2796492890559015| 1.8399097726701694| 1.8144138222208235| 1.8187350816517576|-0.17811912685987313| +-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+ only showing top 20 rows
df_join = df_weather.join(hour_num, ['starttime'], "left")
df_join = df_join.withColumn("count", df_join["count"].cast(IntegerType()))
df_new = df_join.fillna(0, subset='count')
df_final=df_new.orderBy('starttime')
print(df_final.count())
df_final.show()
2924 +-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+ | starttime| SPD| GUS|CLR|SCT|BKN|OVC|OBS|POB| VSB| TEMP| DEWP| SLP| ALT| STP| PCP01|count| +-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+ |2019-04-01 00:00:00| 0.4111950573077732| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.202718242869458|-2.4059857964589435|-0.00771000909058...|-0.03138250547365...|-0.02748200443494122|-0.17811912685987313| 5| |2019-04-01 01:00:00| 2.8743640151934144| 3.3341854908713513| 0| 1| 0| 0| 0| 0|0.4638614904960641| -2.367325382057515|-2.5515963785584366| 0.10097350748358193| 0.07409157039460014| 0.06639344062031857|-0.17811912685987313| 0| |2019-04-01 02:00:00| 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.531932521245572|-2.6244016696081833| 0.1630783740973855| 0.12682860832873663| 0.1289770706571525|-0.17811912685987313| 0| |2019-04-01 03:00:00| 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6244016696081833| 0.2562356740180996| 0.23230268419699088| 0.2384984232216252|-0.17811912685987313| 0| |2019-04-01 04:00:00| 1.6427795362505941|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296| 0.4270240572060726| 0.3905137979993629| 0.39495749831371896|-0.17811912685987313| 0| |2019-04-01 05:00:00| 1.3348834165148888| 3.0033915720566178| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.696539660433629|-2.6972069606579296| 0.5822862237405904| 0.5487249118017536| 0.5514165734058126|-0.17811912685987313| 3| |2019-04-01 06:00:00| 1.3348834165148888| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.7788432300276575| -2.770012251707676| 0.7220221736216527| 0.7069360256041444| 0.7078756484979064|-0.17811912685987313| 16| |2019-04-01 07:00:00| 1.9506756559862992| 3.3341854908713513| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.7788432300276575| -2.770012251707676| 0.9238629901165186| 0.9178841773406529| 0.9112724461176211|-0.17811912685987313| 39| |2019-04-01 08:00:00| 1.3348834165148888| 2.5072006938345175| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296| 1.0014940733837774| 0.9706212152747894| 0.973856076154455|-0.17811912685987313| 63| |2019-04-01 09:00:00| 1.3348834165148888|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.4496289516515435|-2.6972069606579296| 1.1257038066114022| 1.0760952911430437| 1.0833774287189277|-0.17811912685987313| 38| |2019-04-01 10:00:00| 1.0269872967791835| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.2850218124634862|-2.6972069606579296| 1.2499135398390093| 1.2343064049454158| 1.2398365038110215|-0.17811912685987313| 18| |2019-04-01 11:00:00| 1.3348834165148888| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.202718242869458|-2.6972069606579296| 1.3120184064528129| 1.287043442879552| 1.2867742263386424|-0.17811912685987313| 7| |2019-04-01 12:00:00| 2.2585717757220043| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.038111103681401|-2.8428175427574227| 1.3585970564131788| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| 10| |2019-04-01 13:00:00|-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.8735039644933438| -2.915622833807169| 1.3585970564131788| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| 16| |2019-04-01 14:00:00|-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-2.9884281248569153| 1.3741232730666164| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| 17| |2019-04-01 15:00:00| 1.9506756559862992| 3.0033915720566178| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.1340387069564084| 1.3896494897200717| 1.3925175187478065| 1.3962955789031153|-0.17811912685987313| 22| |2019-04-01 16:00:00| 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.2068439980061547| 1.4362281396804375| 1.4452545566819428| 1.4432333014307361|-0.17811912685987313| 32| |2019-04-01 17:00:00| 2.2585717757220043| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.2796492890559015| 1.5914903062149552| 1.5507286325501972| 1.5527546539952088|-0.17811912685987313| 86| |2019-04-01 18:00:00| 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.8735039644933438|-3.2068439980061547| 1.7157000394425623| 1.708939746352569| 1.7092137290873026|-0.17811912685987313| 133| |2019-04-01 19:00:00| 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.9558075340873724|-3.2796492890559015| 1.8399097726701694| 1.8144138222208235| 1.8187350816517576|-0.17811912685987313| 40| +-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+ only showing top 20 rows
from pyspark.sql.types import *
from pyspark.sql import *
from pyspark.sql.types import StructType, StructField, LongType, StringType
from pyspark.sql import Row
from pyspark.sql import Column
import pandas as pd
import numpy as np
import datetime
spark=SparkSession \
.builder \
.appName('my_app_name') \
.getOrCreate()
#here we add more features: hour and wheter it is weekday
pd_final = df_final.toPandas()
pd_hour = [x.hour for x in pd_final['starttime']]
pd_final['hour'] = pd_hour
pd_weekday = [x.weekday() for x in pd_final['starttime']]
pd_final['weekday'] = pd_weekday
pd_final['weekday_end'] = ((pd_final['weekday']//5 == 1).astype(int))
pd_final[pd_final['weekday']==6]
pd_final.drop(columns = ['weekday'], inplace=True)
cols = pd_final.columns
column_names = ['starttime', 'SPD', 'GUS', 'CLR', 'SCT', 'BKN', 'OVC', 'OBS', 'POB',
'VSB', 'TEMP', 'DEWP', 'SLP', 'ALT', 'STP', 'PCP01', 'hour',
'weekday_end', 'count']
pd_final = pd_final[column_names]
df_spark = spark.createDataFrame(pd_final)
df_spark.show()
+-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+----+-----------+-----+ | starttime| SPD| GUS|CLR|SCT|BKN|OVC|OBS|POB| VSB| TEMP| DEWP| SLP| ALT| STP| PCP01|hour|weekday_end|count| +-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+----+-----------+-----+ |2019-04-01 00:00:00| 0.4111950573077732| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.202718242869458|-2.4059857964589435|-0.00771000909058...|-0.03138250547365...|-0.02748200443494122|-0.17811912685987313| 0| 0| 5| |2019-04-01 01:00:00| 2.8743640151934144| 3.3341854908713513| 0| 1| 0| 0| 0| 0|0.4638614904960641| -2.367325382057515|-2.5515963785584366| 0.10097350748358193| 0.07409157039460014| 0.06639344062031857|-0.17811912685987313| 1| 0| 0| |2019-04-01 02:00:00| 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.531932521245572|-2.6244016696081833| 0.1630783740973855| 0.12682860832873663| 0.1289770706571525|-0.17811912685987313| 2| 0| 0| |2019-04-01 03:00:00| 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6244016696081833| 0.2562356740180996| 0.23230268419699088| 0.2384984232216252|-0.17811912685987313| 3| 0| 0| |2019-04-01 04:00:00| 1.6427795362505941|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296| 0.4270240572060726| 0.3905137979993629| 0.39495749831371896|-0.17811912685987313| 4| 0| 0| |2019-04-01 05:00:00| 1.3348834165148888| 3.0033915720566178| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.696539660433629|-2.6972069606579296| 0.5822862237405904| 0.5487249118017536| 0.5514165734058126|-0.17811912685987313| 5| 0| 3| |2019-04-01 06:00:00| 1.3348834165148888| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.7788432300276575| -2.770012251707676| 0.7220221736216527| 0.7069360256041444| 0.7078756484979064|-0.17811912685987313| 6| 0| 16| |2019-04-01 07:00:00| 1.9506756559862992| 3.3341854908713513| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.7788432300276575| -2.770012251707676| 0.9238629901165186| 0.9178841773406529| 0.9112724461176211|-0.17811912685987313| 7| 0| 39| |2019-04-01 08:00:00| 1.3348834165148888| 2.5072006938345175| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296| 1.0014940733837774| 0.9706212152747894| 0.973856076154455|-0.17811912685987313| 8| 0| 63| |2019-04-01 09:00:00| 1.3348834165148888|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.4496289516515435|-2.6972069606579296| 1.1257038066114022| 1.0760952911430437| 1.0833774287189277|-0.17811912685987313| 9| 0| 38| |2019-04-01 10:00:00| 1.0269872967791835| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.2850218124634862|-2.6972069606579296| 1.2499135398390093| 1.2343064049454158| 1.2398365038110215|-0.17811912685987313| 10| 0| 18| |2019-04-01 11:00:00| 1.3348834165148888| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.202718242869458|-2.6972069606579296| 1.3120184064528129| 1.287043442879552| 1.2867742263386424|-0.17811912685987313| 11| 0| 7| |2019-04-01 12:00:00| 2.2585717757220043| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.038111103681401|-2.8428175427574227| 1.3585970564131788| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| 12| 0| 10| |2019-04-01 13:00:00|-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.8735039644933438| -2.915622833807169| 1.3585970564131788| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| 13| 0| 16| |2019-04-01 14:00:00|-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-2.9884281248569153| 1.3741232730666164| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| 14| 0| 17| |2019-04-01 15:00:00| 1.9506756559862992| 3.0033915720566178| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.1340387069564084| 1.3896494897200717| 1.3925175187478065| 1.3962955789031153|-0.17811912685987313| 15| 0| 22| |2019-04-01 16:00:00| 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.2068439980061547| 1.4362281396804375| 1.4452545566819428| 1.4432333014307361|-0.17811912685987313| 16| 0| 32| |2019-04-01 17:00:00| 2.2585717757220043| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.2796492890559015| 1.5914903062149552| 1.5507286325501972| 1.5527546539952088|-0.17811912685987313| 17| 0| 86| |2019-04-01 18:00:00| 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.8735039644933438|-3.2068439980061547| 1.7157000394425623| 1.708939746352569| 1.7092137290873026|-0.17811912685987313| 18| 0| 133| |2019-04-01 19:00:00| 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.9558075340873724|-3.2796492890559015| 1.8399097726701694| 1.8144138222208235| 1.8187350816517576|-0.17811912685987313| 19| 0| 40| +-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+----+-----------+-----+ only showing top 20 rows
from pyspark.sql import SparkSession
#split the dataset tp three set, first one is training set, second one is testing set, the last one is for realtime renewing model set
list_final=df_final.collect()
train_list=list_final[0:1464]
test_list=list_final[1464:2184]
realtime_list=list_final[2184:2924]
a=train_list
b=test_list
c=realtime_list
print(len(b))
print(len(c))
print(a[1400:1464])
print(b[0:50])
print(c[0:50])
720 740 [Row(starttime=datetime.datetime(2019, 5, 29, 8, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.7558606301967902, TEMP=-0.5566468509888874, DEWP=0.2150046813319293, SLP=-1.0169140915649526, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=90), Row(starttime=datetime.datetime(2019, 5, 29, 9, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.5566468509888874, DEWP=0.14219939028218284, SLP=-1.0479665248718455, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=65), Row(starttime=datetime.datetime(2019, 5, 29, 10, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.474343281394859, DEWP=0.14219939028218284, SLP=-1.0169140915649526, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=23), Row(starttime=datetime.datetime(2019, 5, 29, 11, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.1451290030187449, DEWP=0.28780997238167577, SLP=-1.0479665248718455, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=22), Row(starttime=datetime.datetime(2019, 5, 29, 12, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.28780997238167577, SLP=-1.0169140915649526, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=10), Row(starttime=datetime.datetime(2019, 5, 29, 13, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.28780997238167577, SLP=-1.0634927415253008, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=15), Row(starttime=datetime.datetime(2019, 5, 29, 14, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.2150046813319293, SLP=-1.2653335580201843, ALT=-1.2970714158927426, STP=-1.294800512680904, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 5, 29, 15, 0), SPD=1.9506756559862992, GUS=2.3418037344271507, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.2150046813319293, SLP=-1.3895432912477914, ALT=-1.402545491760997, STP=-1.4043218652453588, PCP01=-0.17811912685987313, count=19), Row(starttime=datetime.datetime(2019, 5, 29, 16, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.30973614220680196, DEWP=0.06939409923243636, SLP=-1.3895432912477914, ALT=-1.402545491760997, STP=-1.4043218652453588, PCP01=-0.17811912685987313, count=38), Row(starttime=datetime.datetime(2019, 5, 29, 17, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.7887308313515473, TEMP=-0.6389504205829161, DEWP=0.14219939028218284, SLP=-1.1877024747529255, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=6.460378667254811, count=42), Row(starttime=datetime.datetime(2019, 5, 29, 18, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.7212539901769446, DEWP=0.28780997238167577, SLP=-1.0479665248718455, ALT=-1.2970714158927426, STP=-1.294800512680904, PCP01=4.046379469394926, count=10), Row(starttime=datetime.datetime(2019, 5, 29, 19, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-3.1953048715824988, TEMP=-0.7212539901769446, DEWP=0.28780997238167577, SLP=-1.3119122079805325, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=8.874377865114695, count=11), Row(starttime=datetime.datetime(2019, 5, 29, 20, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.569008710658693, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-1.0324403082184077, ALT=-1.2970714158927426, STP=-1.294800512680904, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 5, 29, 21, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.9755827508896444, TEMP=-0.8858611293650016, DEWP=0.14219939028218284, SLP=-1.0324403082184077, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=11), Row(starttime=datetime.datetime(2019, 5, 29, 22, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.569008710658693, TEMP=-0.8035575597709731, DEWP=0.14219939028218284, SLP=-1.0013878749114973, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=7), Row(starttime=datetime.datetime(2019, 5, 29, 23, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.9755827508896444, TEMP=-0.8035575597709731, DEWP=0.14219939028218284, SLP=-1.0479665248718455, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=4), Row(starttime=datetime.datetime(2019, 5, 30, 0, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.8035575597709731, DEWP=0.14219939028218284, SLP=-1.0634927415253008, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 5, 30, 1, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-1.1100713914856666, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 5, 30, 2, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-1.0634927415253008, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 5, 30, 3, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-1.0324403082184077, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 5, 30, 4, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-1.0013878749114973, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 5, 30, 5, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-0.954809224951149, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=7), Row(starttime=datetime.datetime(2019, 5, 30, 6, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-0.8616519250304349, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=25), Row(starttime=datetime.datetime(2019, 5, 30, 7, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.7212539901769446, DEWP=0.2150046813319293, SLP=-0.8461257083769795, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=58), Row(starttime=datetime.datetime(2019, 5, 30, 8, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.5566468509888874, DEWP=0.36061526343142225, SLP=-0.7063897584959171, ALT=-0.7169639986173161, STP=-0.7159019348401501, PCP01=-0.17811912685987313, count=82), Row(starttime=datetime.datetime(2019, 5, 30, 9, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.5566468509888874, DEWP=0.36061526343142225, SLP=-0.7219159751493724, ALT=-0.7169639986173161, STP=-0.7159019348401501, PCP01=-0.17811912685987313, count=72), Row(starttime=datetime.datetime(2019, 5, 30, 10, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.1624346704277415, TEMP=-0.30973614220680196, DEWP=0.36061526343142225, SLP=-0.7529684084562653, ALT=-0.7697010365514526, STP=-0.7628396573677888, PCP01=-0.17811912685987313, count=23), Row(starttime=datetime.datetime(2019, 5, 30, 11, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.1624346704277415, TEMP=-0.22743257261277341, DEWP=0.36061526343142225, SLP=-0.7995470584166313, ALT=-0.7697010365514526, STP=-0.7628396573677888, PCP01=-0.17811912685987313, count=18), Row(starttime=datetime.datetime(2019, 5, 30, 12, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.7558606301967902, TEMP=0.01947813616931213, DEWP=0.43342055448116873, SLP=-0.8616519250304349, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=27), Row(starttime=datetime.datetime(2019, 5, 30, 13, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.3492865899658387, TEMP=0.10178170576334064, DEWP=0.43342055448116873, SLP=-0.9392830082976936, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=20), Row(starttime=datetime.datetime(2019, 5, 30, 14, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.7558606301967902, TEMP=-0.22743257261277341, DEWP=0.36061526343142225, SLP=-1.0169140915649526, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=23), Row(starttime=datetime.datetime(2019, 5, 30, 15, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.7558606301967902, TEMP=-0.22743257261277341, DEWP=0.43342055448116873, SLP=-1.1255976081391044, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=21), Row(starttime=datetime.datetime(2019, 5, 30, 16, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.7558606301967902, TEMP=-0.22743257261277341, DEWP=0.43342055448116873, SLP=-1.2187549080598183, ALT=-1.1915973400244697, STP=-1.1852791601164312, PCP01=-0.17811912685987313, count=45), Row(starttime=datetime.datetime(2019, 5, 30, 17, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.8700456393977376, TEMP=-0.39203971180083047, DEWP=0.5062258455309152, SLP=-1.3119122079805325, ALT=-1.3498084538268604, STP=-1.3573841427177378, PCP01=1.0288804720700693, count=56), Row(starttime=datetime.datetime(2019, 5, 30, 18, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=1, POB=0, VSB=-3.3985918916979743, TEMP=-0.39203971180083047, DEWP=0.5062258455309152, SLP=-1.2808597746736219, ALT=-1.3498084538268604, STP=-1.3573841427177378, PCP01=8.270878065649725, count=4), Row(starttime=datetime.datetime(2019, 5, 30, 19, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.7887308313515473, TEMP=-0.474343281394859, DEWP=0.5062258455309152, SLP=-1.3895432912477914, ALT=-1.2970714158927426, STP=-1.294800512680904, PCP01=2.235880071000012, count=12), Row(starttime=datetime.datetime(2019, 5, 30, 20, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.8700456393977376, TEMP=-0.39203971180083047, DEWP=0.5790311365806616, SLP=-1.3895432912477914, ALT=-1.402545491760997, STP=-1.4043218652453588, PCP01=0.4253806726050981, count=3), Row(starttime=datetime.datetime(2019, 5, 30, 21, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.39203971180083047, DEWP=0.5790311365806616, SLP=-1.3584908579408808, ALT=-1.4552825296951146, STP=-1.4512595877729977, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 5, 30, 22, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.30973614220680196, DEWP=0.6518364276304082, SLP=-1.451648157861595, ALT=-1.402545491760997, STP=-1.4043218652453588, PCP01=-0.17811912685987313, count=4), Row(starttime=datetime.datetime(2019, 5, 30, 23, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.7887308313515473, TEMP=-0.30973614220680196, DEWP=0.6518364276304082, SLP=-1.5137530244753985, ALT=-1.4552825296951146, STP=-1.4512595877729977, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 5, 31, 0, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.8700456393977376, TEMP=-0.1451290030187449, DEWP=0.7974470097299011, SLP=-1.5603316744357645, ALT=-1.4552825296951146, STP=-1.4512595877729977, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 5, 31, 1, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-3.1953048715824988, TEMP=-0.1451290030187449, DEWP=0.7246417186801546, SLP=-1.5292792411288538, ALT=-1.7189677193657595, STP=-1.7172400154295462, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 5, 31, 2, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.7887308313515473, TEMP=-0.1451290030187449, DEWP=0.7974470097299011, SLP=-1.5603316744357645, ALT=-1.5080195676292512, STP=-1.5138432178098316, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 5, 31, 3, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.22743257261277341, DEWP=0.7246417186801546, SLP=-1.5292792411288538, ALT=-1.5607566055633688, STP=-1.5607809403374526, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 5, 31, 4, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.30973614220680196, DEWP=0.6518364276304082, SLP=-1.4671743745150503, ALT=-1.4552825296951146, STP=-1.4512595877729977, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 5, 31, 5, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.569008710658693, TEMP=-0.30973614220680196, DEWP=0.6518364276304082, SLP=-1.3274384246339879, ALT=-1.3498084538268604, STP=-1.3573841427177378, PCP01=-0.17811912685987313, count=6), Row(starttime=datetime.datetime(2019, 5, 31, 6, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.22743257261277341, DEWP=0.6518364276304082, SLP=-1.2187549080598183, ALT=-1.2443343779586062, STP=-1.247862790153265, PCP01=-0.17811912685987313, count=26), Row(starttime=datetime.datetime(2019, 5, 31, 7, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.7246417186801546, SLP=-1.1566500414460148, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=52), Row(starttime=datetime.datetime(2019, 5, 31, 8, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.6518364276304082, SLP=-1.0324403082184077, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=90), Row(starttime=datetime.datetime(2019, 5, 31, 9, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.5062258455309152, SLP=-1.0013878749114973, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=64), Row(starttime=datetime.datetime(2019, 5, 31, 10, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.5062258455309152, SLP=-0.9237567916442384, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=25), Row(starttime=datetime.datetime(2019, 5, 31, 11, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.36061526343142225, SLP=-0.8771781416838901, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=19), Row(starttime=datetime.datetime(2019, 5, 31, 12, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.36061526343142225, SLP=-0.8771781416838901, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 5, 31, 13, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.8425138321095973, DEWP=0.2150046813319293, SLP=-0.9392830082976936, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=25), Row(starttime=datetime.datetime(2019, 5, 31, 14, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.2150046813319293, SLP=-1.0169140915649526, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=26), Row(starttime=datetime.datetime(2019, 5, 31, 15, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.2150046813319293, SLP=-1.0945451748322113, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=24), Row(starttime=datetime.datetime(2019, 5, 31, 16, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.14219939028218284, SLP=-1.1100713914856666, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=66), Row(starttime=datetime.datetime(2019, 5, 31, 17, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.2150046813319293, SLP=-1.1100713914856666, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=124), Row(starttime=datetime.datetime(2019, 5, 31, 18, 0), SPD=1.6427795362505941, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.2150046813319293, SLP=-1.0790189581787561, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=100), Row(starttime=datetime.datetime(2019, 5, 31, 19, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.14219939028218284, SLP=-1.0013878749114973, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=40), Row(starttime=datetime.datetime(2019, 5, 31, 20, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5956031233275118, DEWP=0.2150046813319293, SLP=-0.9237567916442384, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=11), Row(starttime=datetime.datetime(2019, 5, 31, 21, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.2150046813319293, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=10), Row(starttime=datetime.datetime(2019, 5, 31, 22, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.2150046813319293, SLP=-0.8461257083769795, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 5, 31, 23, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.4309959841394547, DEWP=0.2150046813319293, SLP=-0.8461257083769795, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=8)] [Row(starttime=datetime.datetime(2019, 6, 1, 0, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.28780997238167577, SLP=-0.7995470584166313, ALT=-0.7697010365514526, STP=-0.7628396573677888, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 6, 1, 1, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.2663888449513977, DEWP=0.28780997238167577, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 6, 1, 2, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.36061526343142225, SLP=-0.9392830082976936, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=2), Row(starttime=datetime.datetime(2019, 6, 1, 3, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.36061526343142225, SLP=-0.8771781416838901, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 6, 1, 4, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.22743257261277341, DEWP=0.43342055448116873, SLP=-0.7374421918028101, ALT=-0.7169639986173161, STP=-0.7159019348401501, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 6, 1, 5, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.22743257261277341, DEWP=0.28780997238167577, SLP=-0.6287586752286582, ALT=-0.6114899227490619, STP=-0.6063805822756951, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 6, 1, 6, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.22743257261277341, DEWP=0.28780997238167577, SLP=-0.5511275919613994, ALT=-0.5587528848149441, STP=-0.5594428597480563, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 6, 1, 7, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.06939409923243636, SLP=-0.5821800252682924, ALT=-0.6114899227490619, STP=-0.6063805822756951, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 6, 1, 8, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.06939409923243636, SLP=-0.4890227253475959, ALT=-0.5060158468808076, STP=-0.5125051372204353, PCP01=-0.17811912685987313, count=10), Row(starttime=datetime.datetime(2019, 6, 1, 9, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.2663888449513977, DEWP=-0.003411191817310116, SLP=-0.5200751586544888, ALT=-0.5587528848149441, STP=-0.5594428597480563, PCP01=-0.17811912685987313, count=11), Row(starttime=datetime.datetime(2019, 6, 1, 10, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.06939409923243636, SLP=-0.5356013753079442, ALT=-0.5587528848149441, STP=-0.5594428597480563, PCP01=-0.17811912685987313, count=13), Row(starttime=datetime.datetime(2019, 6, 1, 11, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=0.14219939028218284, SLP=-0.5511275919613994, ALT=-0.5587528848149441, STP=-0.5594428597480563, PCP01=-0.17811912685987313, count=20), Row(starttime=datetime.datetime(2019, 6, 1, 12, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.14219939028218284, SLP=-0.6753373251890066, ALT=-0.6642269606831984, STP=-0.668964212312529, PCP01=-0.17811912685987313, count=22), Row(starttime=datetime.datetime(2019, 6, 1, 13, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=-0.003411191817310116, SLP=-0.8150732750700689, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=20), Row(starttime=datetime.datetime(2019, 6, 1, 14, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.14219939028218284, SLP=-0.8771781416838901, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=28), Row(starttime=datetime.datetime(2019, 6, 1, 15, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.8425138321095973, DEWP=0.2150046813319293, SLP=-0.954809224951149, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=19), Row(starttime=datetime.datetime(2019, 6, 1, 16, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.14219939028218284, SLP=-0.8927043583373278, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=9), Row(starttime=datetime.datetime(2019, 6, 1, 17, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.4309959841394547, DEWP=0.2150046813319293, SLP=-0.9392830082976936, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=21), Row(starttime=datetime.datetime(2019, 6, 1, 18, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.2663888449513977, DEWP=0.43342055448116873, SLP=-0.954809224951149, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 6, 1, 19, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.2663888449513977, DEWP=0.36061526343142225, SLP=-0.9082305749907831, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=22), Row(starttime=datetime.datetime(2019, 6, 1, 20, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.0979448542882077, TEMP=0.10178170576334064, DEWP=0.43342055448116873, SLP=-0.8771781416838901, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=10), Row(starttime=datetime.datetime(2019, 6, 1, 21, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.3492865899658387, TEMP=0.01947813616931213, DEWP=0.5062258455309152, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 6, 1, 22, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.5062258455309152, SLP=-0.8616519250304349, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 6, 1, 23, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.5062258455309152, SLP=-0.8616519250304349, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=4), Row(starttime=datetime.datetime(2019, 6, 2, 0, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.18408527535736915, DEWP=0.5062258455309152, SLP=-0.9703354416045866, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=4), Row(starttime=datetime.datetime(2019, 6, 2, 1, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.3492865899658387, TEMP=0.01947813616931213, DEWP=0.5790311365806616, SLP=-1.0945451748322113, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 6, 2, 2, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.1624346704277415, TEMP=0.10178170576334064, DEWP=0.6518364276304082, SLP=-1.1877024747529255, ALT=-1.1915973400244697, STP=-1.1852791601164312, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 6, 2, 3, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.569008710658693, TEMP=-0.06282543342471639, DEWP=0.6518364276304082, SLP=-1.2653335580201843, ALT=-1.2443343779586062, STP=-1.247862790153265, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 6, 2, 4, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.1624346704277415, TEMP=-0.06282543342471639, DEWP=0.6518364276304082, SLP=-1.2653335580201843, ALT=-1.2970714158927426, STP=-1.294800512680904, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 6, 2, 5, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.1451290030187449, DEWP=0.6518364276304082, SLP=-1.1877024747529255, ALT=-1.1915973400244697, STP=-1.1852791601164312, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 6, 2, 6, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.22743257261277341, DEWP=0.6518364276304082, SLP=-1.1721762580994701, ALT=-1.1915973400244697, STP=-1.1852791601164312, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 6, 2, 7, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.9755827508896444, TEMP=0.01947813616931213, DEWP=0.6518364276304082, SLP=-1.1877024747529255, ALT=-1.1915973400244697, STP=-1.1852791601164312, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 6, 2, 8, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.9755827508896444, TEMP=0.18408527535736915, DEWP=0.6518364276304082, SLP=-1.1255976081391044, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 6, 2, 9, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.0979448542882077, TEMP=0.4309959841394547, DEWP=0.43342055448116873, SLP=-1.0945451748322113, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=9), Row(starttime=datetime.datetime(2019, 6, 2, 10, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5956031233275118, DEWP=0.36061526343142225, SLP=-1.1100713914856666, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 6, 2, 11, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.8425138321095973, DEWP=0.28780997238167577, SLP=-1.1411238247925597, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=12), Row(starttime=datetime.datetime(2019, 6, 2, 12, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.5062258455309152, SLP=-1.2653335580201843, ALT=-1.2443343779586062, STP=-1.247862790153265, PCP01=-0.17811912685987313, count=13), Row(starttime=datetime.datetime(2019, 6, 2, 13, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=0.2150046813319293, SLP=-1.3895432912477914, ALT=-1.402545491760997, STP=-1.4043218652453588, PCP01=-0.17811912685987313, count=28), Row(starttime=datetime.datetime(2019, 6, 2, 14, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.14219939028218284, SLP=-1.5292792411288538, ALT=-1.5080195676292512, STP=-1.5138432178098316, PCP01=-0.17811912685987313, count=24), Row(starttime=datetime.datetime(2019, 6, 2, 15, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=0.36061526343142225, SLP=-1.5913841077426574, ALT=-1.6134936434975053, STP=-1.6077186628650912, PCP01=-0.17811912685987313, count=20), Row(starttime=datetime.datetime(2019, 6, 2, 16, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.36061526343142225, SLP=-1.622436541049568, ALT=-1.6134936434975053, STP=-1.6077186628650912, PCP01=-0.17811912685987313, count=16), Row(starttime=datetime.datetime(2019, 6, 2, 17, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.8700456393977376, TEMP=0.6779066929215403, DEWP=0.5790311365806616, SLP=-1.6534889743564785, ALT=-1.5607566055633688, STP=-1.5607809403374526, PCP01=-0.17811912685987313, count=16), Row(starttime=datetime.datetime(2019, 6, 2, 18, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.0979448542882077, TEMP=0.3486924145454262, DEWP=0.8702523007796475, SLP=-1.6069103243961127, ALT=-1.6134936434975053, STP=-1.6077186628650912, PCP01=-0.17811912685987313, count=18), Row(starttime=datetime.datetime(2019, 6, 2, 19, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.5790311365806616, SLP=-1.5758578910892198, ALT=-1.5607566055633688, STP=-1.5607809403374526, PCP01=-0.17811912685987313, count=4), Row(starttime=datetime.datetime(2019, 6, 2, 20, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.5790311365806616, SLP=-1.4827005911685056, ALT=-1.5080195676292512, STP=-1.5138432178098316, PCP01=-0.17811912685987313, count=12), Row(starttime=datetime.datetime(2019, 6, 2, 21, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.8700456393977376, TEMP=0.3486924145454262, DEWP=0.7246417186801546, SLP=-1.0945451748322113, ALT=-1.4552825296951146, STP=-1.4512595877729977, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 6, 2, 22, 0), SPD=0.7190911770434785, GUS=3.1687885314639845, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-3.4799066997441646, TEMP=-0.22743257261277341, DEWP=0.6518364276304082, SLP=-1.0790189581787561, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=13.702376260834466, count=3), Row(starttime=datetime.datetime(2019, 6, 2, 23, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.39203971180083047, DEWP=0.2150046813319293, SLP=-1.0013878749114973, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 6, 3, 0, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.39203971180083047, DEWP=-0.14902177391680307, SLP=-0.985861658258042, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=2), Row(starttime=datetime.datetime(2019, 6, 3, 1, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.39203971180083047, DEWP=-0.294632356016296, SLP=-1.0013878749114973, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=2)] [Row(starttime=datetime.datetime(2019, 7, 1, 0, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.18408527535736915, DEWP=0.06939409923243636, SLP=-0.411391642080337, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2), Row(starttime=datetime.datetime(2019, 7, 1, 1, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.14219939028218284, SLP=-0.3492867754665158, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 7, 1, 2, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.14219939028218284, SLP=-0.2871819088527123, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 7, 1, 3, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.06939409923243636, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 7, 1, 4, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.06939409923243636, SLP=-0.24060325889236406, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 7, 1, 5, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.1451290030187449, DEWP=0.14219939028218284, SLP=-0.20955082558545346, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 7, 1, 6, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.2150046813319293, SLP=-0.19402460893199816, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=16), Row(starttime=datetime.datetime(2019, 7, 1, 7, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.2150046813319293, SLP=-0.14744595897164992, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=52), Row(starttime=datetime.datetime(2019, 7, 1, 8, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.36061526343142225, SLP=-0.13191974231819462, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=69), Row(starttime=datetime.datetime(2019, 7, 1, 9, 0), SPD=1.9506756559862992, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.28780997238167577, SLP=-0.08534109235784636, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=54), Row(starttime=datetime.datetime(2019, 7, 1, 10, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=0.36061526343142225, SLP=-0.10086730901130166, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=21), Row(starttime=datetime.datetime(2019, 7, 1, 11, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.16297217562510521, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=15), Row(starttime=datetime.datetime(2019, 7, 1, 12, 0), SPD=1.3348834165148888, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.2150046813319293, SLP=-0.1784983922785605, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=15), Row(starttime=datetime.datetime(2019, 7, 1, 13, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.2150046813319293, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 7, 1, 14, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.2150046813319293, SLP=-0.3337605588130782, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=12), Row(starttime=datetime.datetime(2019, 7, 1, 15, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.14219939028218284, SLP=-0.3182343421596229, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=27), Row(starttime=datetime.datetime(2019, 7, 1, 16, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.06939409923243636, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=47), Row(starttime=datetime.datetime(2019, 7, 1, 17, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.06939409923243636, SLP=-0.44244407538722996, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=165), Row(starttime=datetime.datetime(2019, 7, 1, 18, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.47349650869414056, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=128), Row(starttime=datetime.datetime(2019, 7, 1, 19, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.06939409923243636, SLP=-0.4579702920406853, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=36), Row(starttime=datetime.datetime(2019, 7, 1, 20, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.0979448542882077, TEMP=1.089424540891683, DEWP=0.2150046813319293, SLP=-0.4579702920406853, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=18), Row(starttime=datetime.datetime(2019, 7, 1, 21, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.14219939028218284, SLP=-0.3803392087734264, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=7), Row(starttime=datetime.datetime(2019, 7, 1, 22, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=0.14219939028218284, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=11), Row(starttime=datetime.datetime(2019, 7, 1, 23, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.36061526343142225, SLP=-0.3803392087734264, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=7), Row(starttime=datetime.datetime(2019, 7, 2, 0, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5956031233275118, DEWP=0.36061526343142225, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2), Row(starttime=datetime.datetime(2019, 7, 2, 1, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.36061526343142225, SLP=-0.411391642080337, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 7, 2, 2, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.43342055448116873, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 7, 2, 3, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.43342055448116873, SLP=-0.4269178587337747, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2), Row(starttime=datetime.datetime(2019, 7, 2, 4, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.4309959841394547, DEWP=0.43342055448116873, SLP=-0.47349650869414056, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 7, 2, 5, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.4309959841394547, DEWP=0.36061526343142225, SLP=-0.5511275919613994, ALT=-0.5587528848149441, STP=-0.5594428597480563, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 7, 2, 6, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.4309959841394547, DEWP=0.43342055448116873, SLP=-0.5045489420010335, ALT=-0.5060158468808076, STP=-0.5125051372204353, PCP01=-0.17811912685987313, count=23), Row(starttime=datetime.datetime(2019, 7, 2, 7, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5956031233275118, DEWP=0.5062258455309152, SLP=-0.5200751586544888, ALT=-0.5060158468808076, STP=-0.5125051372204353, PCP01=-0.17811912685987313, count=52), Row(starttime=datetime.datetime(2019, 7, 2, 8, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.5062258455309152, SLP=-0.5977062419217477, ALT=-0.6114899227490619, STP=-0.6063805822756951, PCP01=-0.17811912685987313, count=102), Row(starttime=datetime.datetime(2019, 7, 2, 9, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=0.5062258455309152, SLP=-0.6753373251890066, ALT=-0.6642269606831984, STP=-0.668964212312529, PCP01=-0.17811912685987313, count=66), Row(starttime=datetime.datetime(2019, 7, 2, 10, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.43342055448116873, SLP=-0.7219159751493724, ALT=-0.7169639986173161, STP=-0.7159019348401501, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 7, 2, 11, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.5062258455309152, SLP=-0.7219159751493724, ALT=-0.7169639986173161, STP=-0.7159019348401501, PCP01=-0.17811912685987313, count=14), Row(starttime=datetime.datetime(2019, 7, 2, 12, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.5062258455309152, SLP=-0.7995470584166313, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 7, 2, 13, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.5062258455309152, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=15), Row(starttime=datetime.datetime(2019, 7, 2, 14, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.7246417186801546, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=19), Row(starttime=datetime.datetime(2019, 7, 2, 15, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.7246417186801546, SLP=-0.954809224951149, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=25), Row(starttime=datetime.datetime(2019, 7, 2, 16, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.418638819267797, DEWP=1.0158628828791405, SLP=-1.0479665248718455, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=44), Row(starttime=datetime.datetime(2019, 7, 2, 17, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=1.0158628828791405, SLP=-1.0790189581787561, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=112), Row(starttime=datetime.datetime(2019, 7, 2, 18, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.418638819267797, DEWP=0.9430575918293941, SLP=-1.0945451748322113, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=122), Row(starttime=datetime.datetime(2019, 7, 2, 19, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.418638819267797, DEWP=0.9430575918293941, SLP=-1.0634927415253008, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=28), Row(starttime=datetime.datetime(2019, 7, 2, 20, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.8702523007796475, SLP=-1.0790189581787561, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=14), Row(starttime=datetime.datetime(2019, 7, 2, 21, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.9430575918293941, SLP=-0.9703354416045866, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=13), Row(starttime=datetime.datetime(2019, 7, 2, 22, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=1.2342787560283799, SLP=-0.9237567916442384, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=1.0288804720700693, count=4), Row(starttime=datetime.datetime(2019, 7, 2, 23, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=1.088668173928887, SLP=-0.8616519250304349, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 7, 3, 0, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=1.088668173928887, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 7, 3, 1, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=1.0158628828791405, SLP=-0.8616519250304349, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=1)]
from pyspark.sql.types import *
from pyspark.sql import *
from pyspark.sql.types import StructType, StructField, LongType, StringType
from pyspark.sql import Row
from pyspark.sql import Column
import pandas as pd
import numpy as np
train_df=spark.createDataFrame(train_list)
test_df=spark.createDataFrame(test_list)
realtime_df=spark.createDataFrame(realtime_list)
print(train_df.count(),test_df.count(),realtime_df.count())
train_df.cache()
test_df.cache()
realtime_df.cache()
1464 720 740
DataFrame[starttime: timestamp, SPD: double, GUS: double, CLR: bigint, SCT: bigint, BKN: bigint, OVC: bigint, OBS: bigint, POB: bigint, VSB: double, TEMP: double, DEWP: double, SLP: double, ALT: double, STP: double, PCP01: double, count: bigint]
#from pyspark.sql.functions import hour, minute, second
#train_df=train_df.withColumn('starttime', hour(train_df.starttime))
#hour_df=train_df.select(hour('starttime').alias('hour'))
#print(type(hour_df))
#hour_df.show(5)
train_df=train_df.drop("starttime")
test_df=test_df.drop("starttime")
realtime_df=realtime_df.drop("starttime")
train_df.show()
test_df.show()
realtime_df.show()
+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+ | SPD| GUS|CLR|SCT|BKN|OVC|OBS|POB| VSB| TEMP| DEWP| SLP| ALT| STP| PCP01|count| +-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+ | 0.4111950573077732| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.202718242869458|-2.4059857964589435|-0.00771000909058...|-0.03138250547365...|-0.02748200443494122|-0.17811912685987313| 5| | 2.8743640151934144| 3.3341854908713513| 0| 1| 0| 0| 0| 0|0.4638614904960641| -2.367325382057515|-2.5515963785584366| 0.10097350748358193| 0.07409157039460014| 0.06639344062031857|-0.17811912685987313| 0| | 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.531932521245572|-2.6244016696081833| 0.1630783740973855| 0.12682860832873663| 0.1289770706571525|-0.17811912685987313| 0| | 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6244016696081833| 0.2562356740180996| 0.23230268419699088| 0.2384984232216252|-0.17811912685987313| 0| | 1.6427795362505941|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296| 0.4270240572060726| 0.3905137979993629| 0.39495749831371896|-0.17811912685987313| 0| | 1.3348834165148888| 3.0033915720566178| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.696539660433629|-2.6972069606579296| 0.5822862237405904| 0.5487249118017536| 0.5514165734058126|-0.17811912685987313| 3| | 1.3348834165148888| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.7788432300276575| -2.770012251707676| 0.7220221736216527| 0.7069360256041444| 0.7078756484979064|-0.17811912685987313| 16| | 1.9506756559862992| 3.3341854908713513| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.7788432300276575| -2.770012251707676| 0.9238629901165186| 0.9178841773406529| 0.9112724461176211|-0.17811912685987313| 39| | 1.3348834165148888| 2.5072006938345175| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296| 1.0014940733837774| 0.9706212152747894| 0.973856076154455|-0.17811912685987313| 63| | 1.3348834165148888|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.4496289516515435|-2.6972069606579296| 1.1257038066114022| 1.0760952911430437| 1.0833774287189277|-0.17811912685987313| 38| | 1.0269872967791835| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641|-2.2850218124634862|-2.6972069606579296| 1.2499135398390093| 1.2343064049454158| 1.2398365038110215|-0.17811912685987313| 18| | 1.3348834165148888| 3.6649794096860853| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.202718242869458|-2.6972069606579296| 1.3120184064528129| 1.287043442879552| 1.2867742263386424|-0.17811912685987313| 7| | 2.2585717757220043| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641| -2.038111103681401|-2.8428175427574227| 1.3585970564131788| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| 10| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.8735039644933438| -2.915622833807169| 1.3585970564131788| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| 16| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-2.9884281248569153| 1.3741232730666164| 1.33978048081367| 1.3337119488662812|-0.17811912685987313| 17| | 1.9506756559862992| 3.0033915720566178| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.1340387069564084| 1.3896494897200717| 1.3925175187478065| 1.3962955789031153|-0.17811912685987313| 22| | 1.9506756559862992| 2.672597653241884| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.2068439980061547| 1.4362281396804375| 1.4452545566819428| 1.4432333014307361|-0.17811912685987313| 32| | 2.2585717757220043| 3.1687885314639845| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.7912003948993154|-3.2796492890559015| 1.5914903062149552| 1.5507286325501972| 1.5527546539952088|-0.17811912685987313| 86| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.8735039644933438|-3.2068439980061547| 1.7157000394425623| 1.708939746352569| 1.7092137290873026|-0.17811912685987313| 133| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-1.9558075340873724|-3.2796492890559015| 1.8399097726701694| 1.8144138222208235| 1.8187350816517576|-0.17811912685987313| 40| +-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+ only showing top 20 rows +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-----+ | SPD| GUS|CLR|SCT|BKN|OVC|OBS|POB| VSB| TEMP| DEWP| SLP| ALT| STP| PCP01|count| +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-----+ |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.3486924145454262| 0.28780997238167577|-0.7995470584166313|-0.7697010365514526|-0.7628396573677888|-0.17811912685987313| 5| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.2663888449513977| 0.28780997238167577|-0.8305994917235242|-0.8224380744855891|-0.8254232874046228|-0.17811912685987313| 8| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.10178170576334064| 0.36061526343142225|-0.9392830082976936|-0.9279121503538433|-0.9349446399690776|-0.17811912685987313| 2| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.06282543342471639| 0.36061526343142225|-0.8771781416838901|-0.8751751124197069|-0.8723610099322437|-0.17811912685987313| 1| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.22743257261277341| 0.43342055448116873|-0.7374421918028101|-0.7169639986173161|-0.7159019348401501|-0.17811912685987313| 0| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.22743257261277341| 0.28780997238167577|-0.6287586752286582|-0.6114899227490619|-0.6063805822756951|-0.17811912685987313| 1| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.22743257261277341| 0.28780997238167577|-0.5511275919613994|-0.5587528848149441|-0.5594428597480563|-0.17811912685987313| 5| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.01947813616931213| 0.06939409923243636|-0.5821800252682924|-0.6114899227490619|-0.6063805822756951|-0.17811912685987313| 1| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.10178170576334064| 0.06939409923243636|-0.4890227253475959|-0.5060158468808076|-0.5125051372204353|-0.17811912685987313| 10| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.2663888449513977|-0.00341119181731...|-0.5200751586544888|-0.5587528848149441|-0.5594428597480563|-0.17811912685987313| 11| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.5132995537334832| 0.06939409923243636|-0.5356013753079442|-0.5587528848149441|-0.5594428597480563|-0.17811912685987313| 13| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.6779066929215403| 0.14219939028218284|-0.5511275919613994|-0.5587528848149441|-0.5594428597480563|-0.17811912685987313| 20| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.7602102625155688| 0.14219939028218284|-0.6753373251890066|-0.6642269606831984| -0.668964212312529|-0.17811912685987313| 22| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.0071209712976543|-0.00341119181731...|-0.8150732750700689|-0.8224380744855891|-0.8254232874046228|-0.17811912685987313| 20| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.9248174017036258| 0.14219939028218284|-0.8771781416838901|-0.8751751124197069|-0.8723610099322437|-0.17811912685987313| 28| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.8425138321095973| 0.2150046813319293| -0.954809224951149|-0.9806491882879611|-0.9818823624967165|-0.17811912685987313| 19| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.5132995537334832| 0.14219939028218284|-0.8927043583373278|-0.9279121503538433|-0.9349446399690776|-0.17811912685987313| 9| |-0.2045971821636372|-0.3045476160907193| 0| 1| 0| 0| 0| 0|0.4638614904960641| 0.4309959841394547| 0.2150046813319293|-0.9392830082976936|-0.9279121503538433|-0.9349446399690776|-0.17811912685987313| 21| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.2663888449513977| 0.43342055448116873| -0.954809224951149|-0.9806491882879611|-0.9818823624967165|-0.17811912685987313| 17| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.2663888449513977| 0.36061526343142225|-0.9082305749907831|-0.9279121503538433|-0.9349446399690776|-0.17811912685987313| 22| +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-----+ only showing top 20 rows +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+ | SPD| GUS|CLR|SCT|BKN|OVC|OBS|POB| VSB| TEMP| DEWP| SLP| ALT| STP| PCP01|count| +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+ |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.18408527535736915|0.06939409923243636| -0.411391642080337| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313| 2| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.10178170576334064|0.14219939028218284| -0.3492867754665158|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313| 1| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.01947813616931213|0.14219939028218284| -0.2871819088527123| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313| 1| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.01947813616931213|0.06939409923243636|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 0| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.06282543342471639|0.06939409923243636|-0.24060325889236406|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 1| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| -0.1451290030187449|0.14219939028218284|-0.20955082558545346|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 8| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.06282543342471639| 0.2150046813319293|-0.19402460893199816|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313| 16| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.10178170576334064| 0.2150046813319293|-0.14744595897164992|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 52| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.3486924145454262|0.36061526343142225|-0.13191974231819462|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 69| | 1.9506756559862992|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.5132995537334832|0.28780997238167577|-0.08534109235784636|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313| 54| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.6779066929215403|0.36061526343142225|-0.10086730901130166|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313| 21| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.28780997238167577|-0.16297217562510521|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 15| | 1.3348834165148888|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.089424540891683| 0.2150046813319293| -0.1784983922785605|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313| 15| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684| 0.2150046813319293|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 17| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.25403168007974| 0.2150046813319293| -0.3337605588130782|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313| 12| |-0.2045971821636372|-0.3045476160907193| 0| 1| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684|0.14219939028218284| -0.3182343421596229| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313| 27| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684|0.06939409923243636|-0.39586542542688175| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313| 47| | 1.0269872967791835|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.25403168007974|0.06939409923243636|-0.44244407538722996|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 165| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.28780997238167577|-0.47349650869414056|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 128| | 1.0269872967791835|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.06939409923243636| -0.4579702920406853|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 36| +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+ only showing top 20 rows
#a=train_df
#from pyspark.sql.functions import hour, minute, second
#hour_df=a.select(hour('starttime').alias('hour'))
#hour_df.show()
#from pyspark.sql.functions import lit
#a=a.withColumn('hour',lit(None))
#a=a.withColumn("hour", lit(1))
#a.show()
#print(type(a.count()))
# 1.build the pipeline of desision tree
from pyspark.ml import Pipeline
from pyspark.ml.feature import VectorAssembler, VectorIndexer, StringIndexer
from pyspark.ml.regression import DecisionTreeRegressor
features_column = train_df.columns[:-1]
print(features_column)
vector = VectorAssembler(inputCols=features_column, outputCol="va_features")
vIndexer = VectorIndexer(inputCol="va_features", outputCol="features", maxCategories=10)
dt = DecisionTreeRegressor(featuresCol="features", labelCol="count", seed=1024)
dt_pipeline = Pipeline(stages=[vector, vIndexer, dt])
dt_pipeline.getStages()
['SPD', 'GUS', 'CLR', 'SCT', 'BKN', 'OVC', 'OBS', 'POB', 'VSB', 'TEMP', 'DEWP', 'SLP', 'ALT', 'STP', 'PCP01']
[VectorAssembler_76be220d6dbb, VectorIndexer_d6c3a8f78212, DecisionTreeRegressor_e584d684e8e9]
dtModel = dt_pipeline.fit(train_df)
prediction = dtModel.transform(test_df)
prediction.columns
['SPD', 'GUS', 'CLR', 'SCT', 'BKN', 'OVC', 'OBS', 'POB', 'VSB', 'TEMP', 'DEWP', 'SLP', 'ALT', 'STP', 'PCP01', 'count', 'va_features', 'features', 'prediction']
prediction.select("count", "va_features","features","prediction").show()
+-----+--------------------+--------------------+------------------+ |count| va_features| features| prediction| +-----+--------------------+--------------------+------------------+ | 5|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737| | 8|[0.41119505730777...|[0.41119505730777...|15.254098360655737| | 2|[0.41119505730777...|[0.41119505730777...|15.254098360655737| | 1|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737| | 0|[-1.1282855413707...|[-1.1282855413707...| 15.72347266881029| | 1|[0.41119505730777...|[0.41119505730777...| 15.72347266881029| | 5|[-1.1282855413707...|[-1.1282855413707...| 15.72347266881029| | 1|[-1.1282855413707...|[-1.1282855413707...| 73.0| | 10|[0.71909117704347...|[0.71909117704347...| 73.0| | 11|[-1.1282855413707...|[-1.1282855413707...|23.631067961165048| | 13|[0.71909117704347...|[0.71909117704347...| 73.0| | 20|[-1.1282855413707...|[-1.1282855413707...|15.254098360655737| | 22|[0.71909117704347...|[0.71909117704347...|15.254098360655737| | 20|[-1.1282855413707...|[-1.1282855413707...|23.631067961165048| | 28|[0.41119505730777...|[0.41119505730777...|15.254098360655737| | 19|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737| | 9|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737| | 21|[-0.2045971821636...|[-0.2045971821636...|28.137931034482758| | 17|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737| | 22|[0.71909117704347...|[0.71909117704347...|15.254098360655737| +-----+--------------------+--------------------+------------------+ only showing top 20 rows
from pyspark.ml.evaluation import RegressionEvaluator
evaluator = RegressionEvaluator(labelCol="count", predictionCol="prediction", metricName="rmse")
rmse = evaluator.evaluate(prediction)
rmse
36.87194499840543
#realtime streaming
realtime_list=realtime_df.collect()
newtrain_list=train_df.collect()
july_prediction1_list=[]
for i in range(int(realtime_df.count()/12)):
print(i+1)
realtimepart_list=realtime_list[i*12:(i*12+12)]
realtimepart_df = spark.createDataFrame(realtimepart_list)
prediction = dtModel.transform(realtimepart_df)
prediction.select("count", "prediction").show()
rmse = evaluator.evaluate(prediction)
print(rmse)
july_prediction1_list=july_prediction1_list+prediction.collect()
newtrain_list=newtrain_list + realtimepart_list
newtrain_df = spark.createDataFrame(newtrain_list)
dtModel = dt_pipeline.fit(newtrain_df)
print(type(july_prediction1_list))
print(len(july_prediction1_list))
print(july_prediction1_list[0:20])
july_prediction1_df = spark.createDataFrame(july_prediction1_list)
july_prediction1_df.show()
1 +-----+------------------+ |count| prediction| +-----+------------------+ | 2| 73.0| | 1|15.254098360655737| | 1|15.254098360655737| | 0| 73.0| | 1| 73.0| | 8| 15.72347266881029| | 16|15.254098360655737| | 52|15.254098360655737| | 69|15.254098360655737| | 54|15.254098360655737| | 21|15.254098360655737| | 15|15.254098360655737| +-----+------------------+ 42.616287100960605 2 +-----+------------------+ |count| prediction| +-----+------------------+ | 15| 18.88695652173913| | 17| 18.88695652173913| | 12| 18.88695652173913| | 27|36.416666666666664| | 47| 18.88695652173913| | 165| 18.88695652173913| | 128| 18.88695652173913| | 36| 18.88695652173913| | 18| 18.88695652173913| | 7| 18.88695652173913| | 11| 18.88695652173913| | 7| 18.88695652173913| +-----+------------------+ 53.88122761003702 3 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 2|19.72072072072072| | 3|19.72072072072072| | 1|19.72072072072072| | 2|19.72072072072072| | 0| 33.675| | 5|19.72072072072072| | 23|19.72072072072072| | 52|19.72072072072072| | 102|19.72072072072072| | 66|19.72072072072072| | 17|19.72072072072072| | 14|19.72072072072072| +-----+-----------------+ 32.42078424727159 4 +-----+------------------+ |count| prediction| +-----+------------------+ | 17|20.295081967213115| | 15|20.295081967213115| | 19|20.295081967213115| | 25|20.295081967213115| | 44|20.295081967213115| | 112|20.295081967213115| | 122|20.295081967213115| | 28|20.295081967213115| | 14|20.295081967213115| | 13|20.295081967213115| | 4|20.295081967213115| | 5|20.295081967213115| +-----+------------------+ 40.85543180949201 5 +-----+----------+ |count|prediction| +-----+----------+ | 1| 19.848| | 1| 19.848| | 0| 19.848| | 1| 19.848| | 0| 19.848| | 5| 19.848| | 17| 19.848| | 33| 19.848| | 58| 19.848| | 39| 19.848| | 17| 19.848| | 13| 19.848| +-----+----------+ 18.55747209796277 6 +-----+----------+ |count|prediction| +-----+----------+ | 17| 18.0| | 37| 18.0| | 44| 18.0| | 78| 18.0| | 67| 18.0| | 93| 28.0| | 36| 18.0| | 20| 18.0| | 15| 18.0| | 7| 18.0| | 7| 18.0| | 6| 18.0| +-----+----------+ 31.605642956071414 7 +-----+------------------+ |count| prediction| +-----+------------------+ | 3| 18.10569105691057| | 0| 18.10569105691057| | 0| 18.10569105691057| | 0| 18.10569105691057| | 0| 18.10569105691057| | 0| 18.10569105691057| | 2| 18.10569105691057| | 3| 18.10569105691057| | 2|31.967741935483872| | 5| 46.23529411764706| | 8|31.967741935483872| | 12|31.967741935483872| +-----+------------------+ 22.22314060734218 8 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 14|10.51923076923077| | 9|10.51923076923077| | 18|10.51923076923077| | 19|10.51923076923077| | 19|10.51923076923077| | 16|10.51923076923077| | 14|10.51923076923077| | 12|10.51923076923077| | 11|10.51923076923077| | 3|10.51923076923077| | 15|10.51923076923077| | 7|10.51923076923077| +-----+-----------------+ 5.383711004637931 9 +-----+----------+ |count|prediction| +-----+----------+ | 1| 11.0| | 1| 11.0| | 1| 11.0| | 0| 11.0| | 0| 11.0| | 1| 11.0| | 6| 11.0| | 15| 11.0| | 26| 11.0| | 16| 11.0| | 10| 11.0| | 8| 11.0| +-----+----------+ 8.864724097981467 10 +-----+------------------+ |count| prediction| +-----+------------------+ | 15|18.253658536585366| | 15|18.253658536585366| | 16|18.253658536585366| | 23|18.253658536585366| | 24|18.253658536585366| | 49|18.253658536585366| | 21|18.253658536585366| | 18|18.253658536585366| | 15|18.253658536585366| | 7|18.253658536585366| | 7|18.253658536585366| | 11|18.253658536585366| +-----+------------------+ 10.61145421416695 11 +-----+------------------+ |count| prediction| +-----+------------------+ | 7|18.262672811059907| | 2|18.262672811059907| | 1|18.262672811059907| | 1|18.262672811059907| | 0|18.262672811059907| | 2|18.262672811059907| | 2|18.262672811059907| | 4|18.262672811059907| | 4|18.262672811059907| | 3|18.262672811059907| | 3|18.262672811059907| | 15|18.262672811059907| +-----+------------------+ 15.091905697737033 12 +-----+------------------+ |count| prediction| +-----+------------------+ | 16| 17.29004329004329| | 14| 49.44444444444444| | 17| 49.44444444444444| | 12| 49.44444444444444| | 17| 49.44444444444444| | 12| 49.44444444444444| | 18| 49.44444444444444| | 14|30.920245398773005| | 13|30.920245398773005| | 13|30.920245398773005| | 8|30.920245398773005| | 8|30.920245398773005| +-----+------------------+ 27.591398302049143 13 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 7|18.00796812749004| | 3|18.00796812749004| | 1|18.00796812749004| | 0|18.00796812749004| | 0|18.00796812749004| | 0|18.00796812749004| | 1|18.00796812749004| | 3|18.00796812749004| | 4|18.00796812749004| | 5|18.00796812749004| | 6|18.00796812749004| | 10|18.00796812749004| +-----+-----------------+ 14.991123881509472 14 +-----+------------------+ |count| prediction| +-----+------------------+ | 13|17.338403041825096| | 17|17.338403041825096| | 19|17.338403041825096| | 10|17.338403041825096| | 16|17.338403041825096| | 20|17.338403041825096| | 16|17.338403041825096| | 12|17.338403041825096| | 6|17.338403041825096| | 5|17.338403041825096| | 8|17.338403041825096| | 6|17.338403041825096| +-----+------------------+ 7.137199147827578 15 +-----+----------+ |count|prediction| +-----+----------+ | 0| 17.12| | 0| 17.12| | 0| 17.12| | 0| 17.12| | 0| 17.12| | 3| 17.12| | 18| 17.12| | 31| 17.12| | 51| 17.12| | 32| 17.12| | 30| 17.12| | 3| 17.12| +-----+----------+ 17.303209721513134 16 +-----+------------------+ |count| prediction| +-----+------------------+ | 9|16.989547038327526| | 14|16.989547038327526| | 18|16.989547038327526| | 20|16.989547038327526| | 52|16.989547038327526| | 165|16.989547038327526| | 118|16.989547038327526| | 38|16.989547038327526| | 19|16.989547038327526| | 12|16.989547038327526| | 7|16.989547038327526| | 3|16.989547038327526| +-----+------------------+ 53.37342254999157 17 +-----+----------+ |count|prediction| +-----+----------+ | 0| 22.5| | 1| 22.5| | 0| 22.5| | 0| 22.5| | 0| 22.5| | 7| 22.5| | 20| 22.5| | 48| 22.5| | 106| 22.5| | 68| 22.5| | 13| 22.5| | 20| 22.5| +-----+----------+ 32.30454044454639 18 +-----+------------------+ |count| prediction| +-----+------------------+ | 14|21.949868073878626| | 13|21.949868073878626| | 16|21.949868073878626| | 21|21.949868073878626| | 51|21.949868073878626| | 163|21.949868073878626| | 108|21.949868073878626| | 33|21.949868073878626| | 21|21.949868073878626| | 16|21.949868073878626| | 5|21.949868073878626| | 4|21.949868073878626| +-----+------------------+ 49.23682835102618 19 +-----+------------------+ |count| prediction| +-----+------------------+ | 3|22.465473145780052| | 2|22.465473145780052| | 0|22.465473145780052| | 1|22.465473145780052| | 0|22.465473145780052| | 7|22.465473145780052| | 22|22.465473145780052| | 59|22.465473145780052| | 105|22.465473145780052| | 100|22.465473145780052| | 38|22.465473145780052| | 14|22.465473145780052| +-----+------------------+ 37.614943457385365 20 +-----+------------------+ |count| prediction| +-----+------------------+ | 22|22.667493796526056| | 21|22.667493796526056| | 20|22.667493796526056| | 19|22.667493796526056| | 41|22.667493796526056| | 147|22.667493796526056| | 123|22.667493796526056| | 33|22.667493796526056| | 20|22.667493796526056| | 16|22.667493796526056| | 4|22.667493796526056| | 3|22.667493796526056| +-----+------------------+ 47.23934987472937 21 +-----+------------------+ |count| prediction| +-----+------------------+ | 3|23.142168674698794| | 3|23.142168674698794| | 0|23.142168674698794| | 1|23.142168674698794| | 1|23.142168674698794| | 6|23.142168674698794| | 16|23.142168674698794| | 56|23.142168674698794| | 83|23.142168674698794| | 62|23.142168674698794| | 18|23.142168674698794| | 10|23.142168674698794| +-----+------------------+ 27.45310589308695 22 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 14|8.918367346938776| | 12|8.918367346938776| | 18|8.918367346938776| | 4| 56.0| | 30| 9.5| | 79| 9.5| | 76| 2.25| | 7| 9.5| | 5| 56.0| | 3| 9.5| | 3| 9.5| | 1|60.61290322580645| +-----+-----------------+ 40.57586128977382 23 +-----+------------------+ |count| prediction| +-----+------------------+ | 5|27.333333333333332| | 1|27.333333333333332| | 1|27.333333333333332| | 0|27.333333333333332| | 0|27.333333333333332| | 3|27.333333333333332| | 21|27.333333333333332| | 52|27.333333333333332| | 75| 59.92857142857143| | 67| 59.92857142857143| | 20| 59.92857142857143| | 11| 59.92857142857143| +-----+------------------+ 27.212136592501324 24 +-----+------------------+ |count| prediction| +-----+------------------+ | 19|55.970588235294116| | 16|55.970588235294116| | 25|55.970588235294116| | 31|55.970588235294116| | 52|55.970588235294116| | 128|55.970588235294116| | 67|23.385390428211586| | 35|23.385390428211586| | 21|23.385390428211586| | 8|23.385390428211586| | 11|23.385390428211586| | 8|23.385390428211586| +-----+------------------+ 32.17248234816456 25 +-----+------------------+ |count| prediction| +-----+------------------+ | 3|26.408256880733944| | 2| 28.63362068965517| | 3|26.408256880733944| | 2|26.408256880733944| | 0|26.408256880733944| | 0|26.408256880733944| | 3|26.408256880733944| | 6|26.408256880733944| | 6|26.408256880733944| | 6| 28.63362068965517| | 5|26.408256880733944| | 16|26.408256880733944| +-----+------------------+ 22.832112338121764 26 +-----+------------------+ |count| prediction| +-----+------------------+ | 9|29.141025641025642| | 19|29.141025641025642| | 18|29.141025641025642| | 23|29.141025641025642| | 21|29.141025641025642| | 23|29.141025641025642| | 25|29.141025641025642| | 18|29.141025641025642| | 17|29.141025641025642| | 14|29.141025641025642| | 14|29.141025641025642| | 13|16.632373113854594| +-----+------------------+ 11.308901200182975 27 +-----+------------------+ |count| prediction| +-----+------------------+ | 4|19.624719101123596| | 3|19.624719101123596| | 1|19.624719101123596| | 0|19.624719101123596| | 0|19.624719101123596| | 3|19.624719101123596| | 4|19.624719101123596| | 4|19.624719101123596| | 5|19.624719101123596| | 3|19.624719101123596| | 13| 32.61363636363637| | 13| 32.61363636363637| +-----+------------------+ 17.468902439595574 28 +-----+------------------+ |count| prediction| +-----+------------------+ | 15| 29.5| | 18| 29.5| | 33| 29.5| | 20| 29.5| | 26| 29.5| | 16| 29.0| | 8| 164.0| | 9| 164.0| | 7| 164.0| | 3| 164.0| | 4|15.958579881656805| | 2|28.732217573221757| +-----+------------------+ 91.47625788930519 29 +-----+------------------+ |count| prediction| +-----+------------------+ | 4|17.392290249433106| | 1|17.392290249433106| | 0|17.392290249433106| | 0|17.392290249433106| | 0|17.392290249433106| | 7|17.392290249433106| | 28|17.392290249433106| | 68|17.392290249433106| | 92|17.392290249433106| | 66|17.392290249433106| | 18| 24.27077747989276| | 10|17.392290249433106| +-----+------------------+ 31.833142836045685 30 +-----+------------------+ |count| prediction| +-----+------------------+ | 11|17.579646017699115| | 18|17.579646017699115| | 14|17.579646017699115| | 22|17.579646017699115| | 54|17.579646017699115| | 158|17.579646017699115| | 109|17.579646017699115| | 43|17.579646017699115| | 21|17.579646017699115| | 8|17.579646017699115| | 8|17.579646017699115| | 4|17.579646017699115| +-----+------------------+ 50.41796983934571 31 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 1|20.66176470588235| | 5|20.66176470588235| | 1|20.66176470588235| | 0|20.66176470588235| | 0|20.66176470588235| | 3|20.66176470588235| | 25|20.66176470588235| | 51|20.66176470588235| | 92|20.66176470588235| | 76|20.66176470588235| | 23| 20.625| | 20| 14.25| +-----+-----------------+ 30.70506454157064 32 +-----+------------------+ |count| prediction| +-----+------------------+ | 16|14.523809523809524| | 10|14.523809523809524| | 22| 20.88888888888889| | 19| 63.05555555555556| | 53| 63.05555555555556| | 161| 63.05555555555556| | 105| 63.05555555555556| | 37| 63.05555555555556| | 21| 63.05555555555556| | 20| 63.05555555555556| | 5| 63.05555555555556| | 3| 63.05555555555556| +-----+------------------+ 45.365915275584015 33 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 2| 6.5| | 0| 6.5| | 1|19.06644518272425| | 2|19.06644518272425| | 1|19.06644518272425| | 8|19.06644518272425| | 19|19.06644518272425| | 63| 6.5| | 83| 6.5| | 62| 63.44| | 25|17.32608695652174| | 8|17.32608695652174| +-----+-----------------+ 29.327838733757886 34 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 13|16.47826086956522| | 15|16.47826086956522| | 15|36.23529411764706| | 16|36.23529411764706| | 39|36.23529411764706| | 145|36.23529411764706| | 110|36.23529411764706| | 41|36.23529411764706| | 7| 8.12| | 5| 65.0| | 3| 8.12| | 3| 8.12| +-----+-----------------+ 42.651407458714964 35 +-----+------------------+ |count| prediction| +-----+------------------+ | 2| 19.93632075471698| | 1| 19.93632075471698| | 0| 7.888888888888889| | 0| 7.888888888888889| | 0|27.694560669456067| | 3| 19.93632075471698| | 14| 19.93632075471698| | 47| 19.93632075471698| | 43| 19.93632075471698| | 28| 7.888888888888889| | 7| 7.888888888888889| | 9|27.694560669456067| +-----+------------------+ 18.05764400480881 36 +-----+------------------+ |count| prediction| +-----+------------------+ | 9|20.010526315789473| | 12|19.913953488372094| | 30|20.010526315789473| | 19|19.913953488372094| | 34|20.010526315789473| | 155|20.010526315789473| | 93|20.010526315789473| | 37|20.010526315789473| | 15|20.010526315789473| | 6|20.010526315789473| | 5|20.010526315789473| | 5|20.010526315789473| +-----+------------------+ 45.6348625244084 37 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 2|19.89351851851852| | 5|19.89351851851852| | 0|19.89351851851852| | 0|19.89351851851852| | 0|19.89351851851852| | 6|19.89351851851852| | 14|19.89351851851852| | 40|19.89351851851852| | 58|19.89351851851852| | 59|27.95617529880478| | 15|19.89351851851852| | 17|19.89351851851852| +-----+-----------------+ 20.019630172193395 38 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 7| 13.75| | 24| 25.9| | 31|60.93333333333333| | 43|60.93333333333333| | 72|60.93333333333333| | 91|60.93333333333333| | 51| 25.9| | 30| 25.9| | 17| 25.9| | 8| 25.9| | 2|60.93333333333333| | 8|60.93333333333333| +-----+-----------------+ 28.30602482982495 39 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 6|38.72549019607843| | 1|38.72549019607843| | 2|38.72549019607843| | 0|38.72549019607843| | 1|38.72549019607843| | 1|38.72549019607843| | 1|38.72549019607843| | 2|38.72549019607843| | 4|38.72549019607843| | 10|38.72549019607843| | 12|38.72549019607843| | 11|38.72549019607843| +-----+-----------------+ 34.73106569427291 40 +-----+------------------+ |count| prediction| +-----+------------------+ | 11|60.733333333333334| | 12|60.733333333333334| | 10|60.733333333333334| | 15|60.733333333333334| | 10|60.733333333333334| | 17|60.733333333333334| | 10|60.733333333333334| | 13|60.733333333333334| | 11|60.733333333333334| | 9|60.733333333333334| | 4|60.733333333333334| | 6|60.733333333333334| +-----+------------------+ 50.18027500921054 41 +-----+------------------+ |count| prediction| +-----+------------------+ | 1|15.631578947368421| | 2|15.631578947368421| | 0|15.631578947368421| | 0|15.631578947368421| | 0|15.631578947368421| | 1|26.931506849315067| | 1|15.631578947368421| | 4|15.631578947368421| | 5|15.631578947368421| | 3|15.631578947368421| | 5|15.631578947368421| | 7|15.631578947368421| +-----+------------------+ 14.761671909287497 42 +-----+------------------+ |count| prediction| +-----+------------------+ | 16|18.884318766066837| | 16|18.884318766066837| | 16|18.884318766066837| | 15|18.884318766066837| | 11|52.674418604651166| | 15|18.884318766066837| | 16|18.884318766066837| | 6|18.884318766066837| | 7|18.884318766066837| | 7|18.884318766066837| | 3|18.884318766066837| | 2|18.884318766066837| +-----+------------------+ 15.237239873509608 43 +-----+------------------+ |count| prediction| +-----+------------------+ | 3|18.661654135338345| | 2|18.661654135338345| | 3|18.661654135338345| | 0|18.661654135338345| | 0|18.661654135338345| | 4|18.661654135338345| | 23|18.661654135338345| | 53|18.661654135338345| | 70|18.661654135338345| | 55|18.661654135338345| | 16|18.661654135338345| | 9|18.661654135338345| +-----+------------------+ 24.038407899967844 44 +-----+------------------+ |count| prediction| +-----+------------------+ | 9|25.674107142857142| | 14|25.674107142857142| | 12|50.680851063829785| | 14| 17.2| | 27| 5.4| | 89|50.680851063829785| | 15| 17.2| | 4| 79.0| | 9|20.848484848484848| | 13|50.680851063829785| | 8|50.680851063829785| | 6| 19.0813704496788| +-----+------------------+ 32.962764941777884 45 +-----+------------------+ |count| prediction| +-----+------------------+ | 0| 16.75| | 1| 52.91111111111111| | 0| 6.75| | 1| 6.75| | 0| 6.75| | 1| 79.0| | 0| 6.75| | 10| 6.75| | 77|18.966101694915253| | 63|18.966101694915253| | 15|18.966101694915253| | 17|18.966101694915253| +-----+------------------+ 34.840292200921084 46 +-----+------------------+ |count| prediction| +-----+------------------+ | 16|19.215031315240083| | 13|19.215031315240083| | 18|19.215031315240083| | 22|19.215031315240083| | 31|19.215031315240083| | 116|19.215031315240083| | 138|19.215031315240083| | 41|19.215031315240083| | 20|19.215031315240083| | 8|19.215031315240083| | 11|19.215031315240083| | 6|19.215031315240083| +-----+------------------+ 45.2008310984955 47 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 1|23.27992277992278| | 0| 6.5| | 1|23.27992277992278| | 1|23.27992277992278| | 0|23.27992277992278| | 6|23.27992277992278| | 26|23.27992277992278| | 62|23.27992277992278| | 90|23.27992277992278| | 93|23.27992277992278| | 31|23.27992277992278| | 22|23.27992277992278| +-----+-----------------+ 33.23191773932889 48 +-----+------------------+ |count| prediction| +-----+------------------+ | 28|23.425330812854444| | 26|23.425330812854444| | 36|23.425330812854444| | 36|23.425330812854444| | 54|23.425330812854444| | 118|23.425330812854444| | 110|23.425330812854444| | 57|23.425330812854444| | 22|23.425330812854444| | 16|23.425330812854444| | 6|23.425330812854444| | 6|23.425330812854444| +-----+------------------+ 40.32148181290375 49 +-----+------------------+ |count| prediction| +-----+------------------+ | 3|23.868512110726645| | 6|23.868512110726645| | 2|23.868512110726645| | 2|23.868512110726645| | 2|23.868512110726645| | 6|23.868512110726645| | 21|23.868512110726645| | 54|23.868512110726645| | 89|23.868512110726645| | 71|23.868512110726645| | 24|23.868512110726645| | 13|23.868512110726645| +-----+------------------+ 28.87692612287684 50 +-----+------------------+ |count| prediction| +-----+------------------+ | 13| 25.83783783783784| | 23| 25.83783783783784| | 13| 25.83783783783784| | 30| 25.83783783783784| | 55| 25.83783783783784| | 142| 25.83783783783784| | 112| 25.83783783783784| | 50|19.400437636761488| | 27| 25.83783783783784| | 21|19.400437636761488| | 22|19.400437636761488| | 13|19.400437636761488| +-----+------------------+ 43.885229719754875 51 +-----+------------------+ |count| prediction| +-----+------------------+ | 8| 26.44868035190616| | 0| 26.44868035190616| | 1| 26.44868035190616| | 1|19.462039045553144| | 2|19.462039045553144| | 7|19.462039045553144| | 23|19.462039045553144| | 37|19.462039045553144| | 71|19.462039045553144| | 67| 26.44868035190616| | 18| 26.44868035190616| | 12| 26.44868035190616| +-----+------------------+ 24.814993269455666 52 +-----+------------------+ |count| prediction| +-----+------------------+ | 18|26.543408360128616| | 22|26.543408360128616| | 37|26.543408360128616| | 42| 19.51502145922747| | 80| 19.51502145922747| | 105|26.543408360128616| | 65|26.543408360128616| | 29|26.543408360128616| | 15|26.543408360128616| | 23|26.543408360128616| | 11|26.543408360128616| | 7|26.543408360128616| +-----+------------------+ 32.63013562600372 53 +-----+------------------+ |count| prediction| +-----+------------------+ | 7| 26.75077881619938| | 2| 26.75077881619938| | 2| 26.75077881619938| | 1| 26.75077881619938| | 0|19.692307692307693| | 0| 26.75077881619938| | 5|19.692307692307693| | 4|19.692307692307693| | 8|19.692307692307693| | 12|19.692307692307693| | 10|19.692307692307693| | 21|19.692307692307693| +-----+------------------+ 18.56393029829631 54 +-----+------------------+ |count| prediction| +-----+------------------+ | 18|19.555555555555557| | 13|19.555555555555557| | 13|19.555555555555557| | 23|19.555555555555557| | 25|19.555555555555557| | 21|19.555555555555557| | 25|19.555555555555557| | 15|19.555555555555557| | 11|19.555555555555557| | 11|19.555555555555557| | 11|19.555555555555557| | 3|19.555555555555557| +-----+------------------+ 7.505759105719183 55 +-----+------------------+ |count| prediction| +-----+------------------+ | 2|19.460416666666667| | 1|19.460416666666667| | 2|19.460416666666667| | 0|19.460416666666667| | 0|19.460416666666667| | 1|19.460416666666667| | 1|19.460416666666667| | 0|19.460416666666667| | 4|19.460416666666667| | 6|19.460416666666667| | 10|19.460416666666667| | 10|19.460416666666667| +-----+------------------+ 16.75167396797009 56 +-----+------------------+ |count| prediction| +-----+------------------+ | 13|27.472868217054263| | 14|27.472868217054263| | 21|27.472868217054263| | 5|27.472868217054263| | 0|27.472868217054263| | 0|27.472868217054263| | 0|27.472868217054263| | 0|27.472868217054263| | 0|27.472868217054263| | 0|27.472868217054263| | 0|27.472868217054263| | 0|19.786941580756015| +-----+------------------+ 23.47470694687588 57 +-----+-----------------+ |count| prediction| +-----+-----------------+ | 0|19.77758620689655| | 0|19.77758620689655| | 2|19.77758620689655| | 1|19.77758620689655| | 0|19.77758620689655| | 6|19.77758620689655| | 14|19.77758620689655| | 44|19.77758620689655| | 92|19.77758620689655| | 55|26.54646840148699| | 23|26.54646840148699| | 10|26.54646840148699| +-----+-----------------+ 27.331407981264046 58 +-----+------------------+ |count| prediction| +-----+------------------+ | 19|21.658620689655173| | 10|21.658620689655173| | 16|21.658620689655173| | 19|21.658620689655173| | 47|21.658620689655173| | 148|21.658620689655173| | 135|21.658620689655173| | 41|21.658620689655173| | 25|21.658620689655173| | 5|21.658620689655173| | 6|21.658620689655173| | 3|21.658620689655173| +-----+------------------+ 50.735242343913164 59 +-----+------------------+ |count| prediction| +-----+------------------+ | 1|24.578034682080926| | 1|17.508951406649615| | 1|17.508951406649615| | 0|17.508951406649615| | 0|17.508951406649615| | 8|17.508951406649615| | 24|17.508951406649615| | 46|17.508951406649615| | 94|17.508951406649615| | 47|24.578034682080926| | 19|24.578034682080926| | 19|24.578034682080926| +-----+------------------+ 27.4989229113094 60 +-----+------------------+ |count| prediction| +-----+------------------+ | 11|22.278443113772454| | 11|22.278443113772454| | 25|22.278443113772454| | 11|22.278443113772454| | 42|22.278443113772454| | 158|22.278443113772454| | 113|22.278443113772454| | 34|22.278443113772454| | 23|22.278443113772454| | 8|22.278443113772454| | 10|22.278443113772454| | 5|22.278443113772454| +-----+------------------+ 48.49380771660473 61 +-----+------------------+ |count| prediction| +-----+------------------+ | 0|22.591036414565828| | 1|22.591036414565828| | 1|22.591036414565828| | 3|22.591036414565828| | 0|22.591036414565828| | 6|22.591036414565828| | 24|22.591036414565828| | 61|22.591036414565828| | 103|22.591036414565828| | 64|22.591036414565828| | 20|22.591036414565828| | 12|22.591036414565828| +-----+------------------+ 32.130654665517675 <class 'list'> 732 [Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.18408527535736915, DEWP=0.06939409923243636, SLP=-0.411391642080337, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), prediction=73.0), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.14219939028218284, SLP=-0.3492867754665158, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), prediction=15.254098360655737), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.14219939028218284, SLP=-0.2871819088527123, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), prediction=15.254098360655737), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.06939409923243636, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=0, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=73.0), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.06939409923243636, SLP=-0.24060325889236406, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), prediction=73.0), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.1451290030187449, DEWP=0.14219939028218284, SLP=-0.20955082558545346, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=8, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), prediction=15.72347266881029), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.2150046813319293, SLP=-0.19402460893199816, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=16, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), prediction=15.254098360655737), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.2150046813319293, SLP=-0.14744595897164992, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=52, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), prediction=15.254098360655737), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.36061526343142225, SLP=-0.13191974231819462, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=69, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), prediction=15.254098360655737), Row(SPD=1.9506756559862992, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.28780997238167577, SLP=-0.08534109235784636, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=54, va_features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), prediction=15.254098360655737), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=0.36061526343142225, SLP=-0.10086730901130166, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=21, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), prediction=15.254098360655737), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.16297217562510521, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), prediction=15.254098360655737), Row(SPD=1.3348834165148888, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.2150046813319293, SLP=-0.1784983922785605, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), prediction=18.88695652173913), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.2150046813319293, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=17, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=18.88695652173913), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.2150046813319293, SLP=-0.3337605588130782, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=12, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), prediction=18.88695652173913), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.14219939028218284, SLP=-0.3182343421596229, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=27, va_features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), prediction=36.416666666666664), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.06939409923243636, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=47, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), prediction=18.88695652173913), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.06939409923243636, SLP=-0.44244407538722996, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=165, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), prediction=18.88695652173913), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.47349650869414056, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=128, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), prediction=18.88695652173913), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.06939409923243636, SLP=-0.4579702920406853, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=36, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), prediction=18.88695652173913)] +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+ | SPD| GUS|CLR|SCT|BKN|OVC|OBS|POB| VSB| TEMP| DEWP| SLP| ALT| STP| PCP01|count| va_features| features| prediction| +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+ |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.18408527535736915|0.06939409923243636| -0.411391642080337| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313| 2|[-1.1282855413707...|[-1.1282855413707...| 73.0| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.10178170576334064|0.14219939028218284| -0.3492867754665158|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313| 1|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.01947813616931213|0.14219939028218284| -0.2871819088527123| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313| 1|[0.71909117704347...|[0.71909117704347...|15.254098360655737| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.01947813616931213|0.06939409923243636|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 0|[-1.1282855413707...|[-1.1282855413707...| 73.0| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.06282543342471639|0.06939409923243636|-0.24060325889236406|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 1|[-0.2045971821636...|[-0.2045971821636...| 73.0| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| -0.1451290030187449|0.14219939028218284|-0.20955082558545346|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 8|[0.71909117704347...|[0.71909117704347...| 15.72347266881029| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.06282543342471639| 0.2150046813319293|-0.19402460893199816|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313| 16|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.10178170576334064| 0.2150046813319293|-0.14744595897164992|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 52|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.3486924145454262|0.36061526343142225|-0.13191974231819462|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 69|[-1.1282855413707...|[-1.1282855413707...|15.254098360655737| | 1.9506756559862992|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.5132995537334832|0.28780997238167577|-0.08534109235784636|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313| 54|[1.95067565598629...|[1.95067565598629...|15.254098360655737| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.6779066929215403|0.36061526343142225|-0.10086730901130166|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313| 21|[0.41119505730777...|[0.41119505730777...|15.254098360655737| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.28780997238167577|-0.16297217562510521|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 15|[0.41119505730777...|[0.41119505730777...|15.254098360655737| | 1.3348834165148888|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.089424540891683| 0.2150046813319293| -0.1784983922785605|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313| 15|[1.33488341651488...|[1.33488341651488...| 18.88695652173913| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684| 0.2150046813319293|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 17|[-0.2045971821636...|[-0.2045971821636...| 18.88695652173913| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.25403168007974| 0.2150046813319293| -0.3337605588130782|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313| 12|[0.41119505730777...|[0.41119505730777...| 18.88695652173913| |-0.2045971821636372|-0.3045476160907193| 0| 1| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684|0.14219939028218284| -0.3182343421596229| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313| 27|[-0.2045971821636...|[-0.2045971821636...|36.416666666666664| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684|0.06939409923243636|-0.39586542542688175| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313| 47|[-0.2045971821636...|[-0.2045971821636...| 18.88695652173913| | 1.0269872967791835|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.25403168007974|0.06939409923243636|-0.44244407538722996|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 165|[1.02698729677918...|[1.02698729677918...| 18.88695652173913| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.28780997238167577|-0.47349650869414056|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 128|[0.71909117704347...|[0.71909117704347...| 18.88695652173913| | 1.0269872967791835|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.06939409923243636| -0.4579702920406853|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 36|[1.02698729677918...|[1.02698729677918...| 18.88695652173913| +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+ only showing top 20 rows
df_reg = july_prediction1_df.select("*").toPandas()
df_reg.to_csv('df_regression.csv')
df_reg
| SPD | GUS | CLR | SCT | BKN | OVC | OBS | POB | VSB | TEMP | DEWP | SLP | ALT | STP | PCP01 | count | va_features | features | prediction | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.184085 | 0.069394 | -0.411392 | -0.400542 | -0.402984 | -0.178119 | 2 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 73.000000 |
| 1 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.101782 | 0.142199 | -0.349287 | -0.347805 | -0.340400 | -0.178119 | 1 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 15.254098 |
| 2 | 0.719091 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.019478 | 0.142199 | -0.287182 | -0.295068 | -0.293462 | -0.178119 | 1 | [0.7190911770434785, -0.3045476160907193, 1.0,... | [0.7190911770434785, -0.3045476160907193, 1.0,... | 15.254098 |
| 3 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.019478 | 0.069394 | -0.256129 | -0.242331 | -0.246525 | -0.178119 | 0 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 73.000000 |
| 4 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | -0.062825 | 0.069394 | -0.240603 | -0.242331 | -0.246525 | -0.178119 | 1 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 73.000000 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 727 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.097945 | 0.924817 | 1.161473 | 0.302814 | 0.337777 | 0.332374 | -0.178119 | 61 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 22.591036 |
| 728 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | -0.349287 | 1.089425 | 1.161473 | 0.333867 | 0.337777 | 0.332374 | -0.178119 | 103 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 22.591036 |
| 729 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | -0.349287 | 1.171728 | 1.161473 | 0.333867 | 0.337777 | 0.332374 | -0.178119 | 64 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 22.591036 |
| 730 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | -0.349287 | 1.418639 | 1.161473 | 0.380445 | 0.390514 | 0.394957 | -0.178119 | 20 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 22.591036 |
| 731 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 1.418639 | 1.088668 | 0.364919 | 0.390514 | 0.394957 | -0.178119 | 12 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 22.591036 |
732 rows × 19 columns
#2.build the pipline of gradient boost tree regression
from pyspark.ml.regression import GBTRegressor
gbt = GBTRegressor(labelCol="count", featuresCol="features")
gbt_pipeline = Pipeline(stages=[vector, vIndexer, gbt])
gbt_model = gbt_pipeline.fit(train_df)
gbt_prediction = gbt_model.transform(test_df)
gbt_prediction.select("count", "va_features","features","prediction").show()
gbt_rmse = evaluator.evaluate(gbt_prediction)
gbt_rmse
+-----+--------------------+--------------------+------------------+ |count| va_features| features| prediction| +-----+--------------------+--------------------+------------------+ | 5|[-0.2045971821636...|[-0.2045971821636...|14.167528541903247| | 8|[0.41119505730777...|[0.41119505730777...|15.762970386846868| | 2|[0.41119505730777...|[0.41119505730777...|16.165373708927575| | 1|[-0.2045971821636...|[-0.2045971821636...|16.165373708927575| | 0|[-1.1282855413707...|[-1.1282855413707...|14.318582260299902| | 1|[0.41119505730777...|[0.41119505730777...|12.958096034728644| | 5|[-1.1282855413707...|[-1.1282855413707...|12.958096034728644| | 1|[-1.1282855413707...|[-1.1282855413707...| 83.60026986973979| | 10|[0.71909117704347...|[0.71909117704347...| 67.07887357859991| | 11|[-1.1282855413707...|[-1.1282855413707...|19.105158819305256| | 13|[0.71909117704347...|[0.71909117704347...| 66.01947449524297| | 20|[-1.1282855413707...|[-1.1282855413707...|12.493261239382463| | 22|[0.71909117704347...|[0.71909117704347...|18.392433998218916| | 20|[-1.1282855413707...|[-1.1282855413707...| 16.67009126337221| | 28|[0.41119505730777...|[0.41119505730777...|14.158786115650623| | 19|[-0.2045971821636...|[-0.2045971821636...|14.158786115650623| | 9|[-0.2045971821636...|[-0.2045971821636...|15.287032905821656| | 21|[-0.2045971821636...|[-0.2045971821636...| 29.57280198421807| | 17|[-0.2045971821636...|[-0.2045971821636...| 16.48369429868547| | 22|[0.71909117704347...|[0.71909117704347...| 19.82571943908685| +-----+--------------------+--------------------+------------------+ only showing top 20 rows
38.27806026631097
#realtime streaming of GBTR
realtime_list=realtime_df.collect()
newtrain_list=train_df.collect()
july_prediction2_list=[]
for i in range(int(realtime_df.count()/12)):
print(i+1)
realtimepart_list=realtime_list[i*12:(i*12+12)]
realtimepart_df = spark.createDataFrame(realtimepart_list)
gbt_prediction = gbt_model.transform(realtimepart_df)
gbt_prediction.select("count", "prediction").show()
rmse = evaluator.evaluate(gbt_prediction)
print(rmse)
july_prediction2_list=july_prediction2_list+gbt_prediction.collect()
newtrain_list=newtrain_list + realtimepart_list
newtrain_df = spark.createDataFrame(newtrain_list)
gbt_model = gbt_pipeline.fit(newtrain_df)
print(type(july_prediction2_list))
print(len(july_prediction2_list))
print(july_prediction2_list[0:20])
july_prediction2_df = spark.createDataFrame(july_prediction2_list)
july_prediction2_df.show()
1 +-----+------------------+ |count| prediction| +-----+------------------+ | 2| 61.59404460991423| | 1| 7.778935354101069| | 1|15.966927358090784| | 0| 82.31708145370767| | 1| 82.31708145370767| | 8|18.014751874827045| | 16| 11.58732212625986| | 52| 9.357385562682786| | 69|12.637939746904813| | 54| 25.24644542515382| | 21|11.942369854237393| | 15|12.256402693047209| +-----+------------------+ 44.008672385049046 2 +-----+------------------+ |count| prediction| +-----+------------------+ | 15|23.153034885322995| | 17|17.922527217444244| | 12|17.922527217444244| | 27| 35.94808745540508| | 47|15.677677873198391| | 165|15.677677873198391| | 128|18.427032192739624| | 36|15.677677873198391| | 18| 18.72544654958129| | 7|18.427032192739624| | 11|18.427032192739624| | 7|18.427032192739624| +-----+------------------+ 54.921750883178575 3 +-----+------------------+ |count| prediction| +-----+------------------+ | 2|19.846527437718258| | 3|13.909832811299347| | 1|16.549829045586407| | 2| 30.54390426115427| | 0|31.900978143890647| | 5|17.946698864611374| | 23|17.946698864611374| | 52|21.243397256743226| | 102|28.304206011972752| | 66|21.243397256743226| | 17|21.243397256743226| | 14|21.971403113888442| +-----+------------------+ 30.504974674357197 4 +-----+------------------+ |count| prediction| +-----+------------------+ | 17|27.873722212475727| | 15| 27.15229547121602| | 19|25.828922362070486| | 25|22.075890808794966| | 44| 22.87934191261387| | 112|23.986461882434195| | 122| 27.4425946824497| | 28| 35.23801716355055| | 14|26.529442511048146| | 13|23.986461882434195| | 4|22.687616482472023| | 5| 22.43974587811664| +-----+------------------+ 39.19834033311445 5 +-----+------------------+ |count| prediction| +-----+------------------+ | 1|22.493375535453147| | 1| 8.467205610276096| | 0|15.179736997706383| | 1| 25.77169878479795| | 0|31.392109480593565| | 5|27.166885472527525| | 17|13.080056670186371| | 33|13.080056670186371| | 58| 21.69775339278299| | 39| 15.52173753852034| | 17|23.374977609905645| | 13| 5.938352858406112| +-----+------------------+ 20.837427047512 6 +-----+------------------+ |count| prediction| +-----+------------------+ | 17| 17.31520433765312| | 37|23.344046364450055| | 44| 20.31139539653864| | 78|23.344046364450055| | 67|23.344046364450055| | 93| 33.34404636445006| | 36|25.028298531197883| | 20|24.419405774524314| | 15|14.185499186036171| | 7| 27.98190899588396| | 7|17.409471855873708| | 6|14.965670587162931| +-----+------------------+ 28.82307585648045 7 +-----+------------------+ |count| prediction| +-----+------------------+ | 3| 9.345680160666273| | 0| 16.1849763144682| | 0| 16.1849763144682| | 0| 39.12344561416115| | 0| 33.58753488004096| | 0|17.000659322451188| | 2|18.068532493043453| | 3| 34.65540805063323| | 2| 33.34656163841736| | 5| 45.80968249846542| | 8|29.321078234573953| | 12|26.930609226234417| +-----+------------------+ 25.977192982822434 8 +-----+------------------+ |count| prediction| +-----+------------------+ | 14|11.663619098497314| | 9| 8.918717408844822| | 18|19.159338375312238| | 19|18.535830255961915| | 19| 8.2952092894945| | 16|13.350673791213534| | 14|11.408801906753943| | 12|11.944988215947735| | 11| 5.399946336989276| | 3| 7.503862118884064| | 15| 4.551951896632436| | 7| 4.551951896632436| +-----+------------------+ 5.017914844738803 9 +-----+------------------+ |count| prediction| +-----+------------------+ | 1| 5.89928019933442| | 1| 6.013534693742454| | 1| 6.013534693742454| | 0|14.916755787454331| | 0| 8.284267298406768| | 1| 8.284267298406768| | 6| 12.71872992492693| | 15|7.2173319049034035| | 26|15.695174521184395| | 16| 21.12710921136433| | 10| 20.16838233532268| | 8| 20.16838233532268| +-----+------------------+ 8.699265793557398 10 +-----+------------------+ |count| prediction| +-----+------------------+ | 15| 32.83481257386646| | 15|21.594045009780494| | 16|17.227915575114704| | 23|17.145078023101057| | 24| 24.89777575062174| | 49| 14.14030891507898| | 21| 15.31831975925873| | 18| 30.9996693058868| | 15| 15.31831975925873| | 7| 15.31831975925873| | 7| 15.31831975925873| | 11| 8.717104637712213| +-----+------------------+ 12.774742787185414 11 +-----+------------------+ |count| prediction| +-----+------------------+ | 7|14.807302290051968| | 2|14.251998621748678| | 1|14.251998621748678| | 1|14.251998621748678| | 0| 22.45145603616718| | 2| 22.45145603616718| | 2| 22.45145603616718| | 4| 22.45145603616718| | 4|23.572593704883946| | 3|18.200064606637618| | 3|22.287016712064126| | 15| 24.84550307863883| +-----+------------------+ 16.64931410779361 12 +-----+------------------+ |count| prediction| +-----+------------------+ | 16| 12.59055617278233| | 14| 42.37805944397618| | 17| 50.15338411930086| | 12| 40.11504040917744| | 17| 40.11504040917744| | 12|27.738692357869237| | 18|48.549698223304496| | 14|23.602074662957087| | 13|26.667982781175667| | 13|26.667982781175667| | 8|23.072059239921238| | 8|23.072059239921238| +-----+------------------+ 21.09896880672596 13 +-----+------------------+ |count| prediction| +-----+------------------+ | 7|15.166028568140357| | 3|14.803121778561616| | 1|12.737491296990251| | 0|16.294597491520662| | 0|15.166028568140357| | 0|15.150987625880575| | 1|13.402334593455851| | 3|14.002689476196005| | 4| 13.84930483335112| | 5| 9.591648073154264| | 6|11.770759972328099| | 10|16.957601559574993| +-----+------------------+ 11.34137032853502 14 +-----+------------------+ |count| prediction| +-----+------------------+ | 13| 17.98757579830893| | 17| 42.08298853584196| | 19|34.943391768760826| | 10|12.253477075596907| | 16| 25.27942562272722| | 20| 13.51819417607071| | 16|15.574013754685227| | 12|15.574013754685227| | 6|18.148434080378337| | 5|16.304585555849116| | 8|16.304585555849116| | 6| 35.14920484302878| +-----+------------------+ 13.688191690018488 15 +-----+------------------+ |count| prediction| +-----+------------------+ | 0|13.646863227216727| | 0|13.646863227216727| | 0| 39.37214952154402| | 0| 48.85032011694959| | 0|12.434778194177149| | 3|14.059346857703199| | 18|11.388319803949999| | 31|12.434778194177149| | 51| 11.21150615050385| | 32| 23.54199072777498| | 30|16.797588571643935| | 3|16.797588571643935| +-----+------------------+ 24.1413331658727 16 +-----+------------------+ |count| prediction| +-----+------------------+ | 9|22.152214214334066| | 14|17.774530034431333| | 18|18.359825973817408| | 20|16.617122673928947| | 52|15.084641661316937| | 165|13.229524505303456| | 118|14.762005517915464| | 38|15.958227962705529| | 19|13.731078417040813| | 12|13.245391909555444| | 7|12.534855972250748| | 3|12.534855972250748| +-----+------------------+ 54.688973543228045 17 +-----+------------------+ |count| prediction| +-----+------------------+ | 0|14.035021075591773| | 1|14.035021075591773| | 0| 21.23362671797738| | 0|15.599427921604345| | 0| 21.00712118861493| | 7|13.787348398783951| | 20|13.662107153130492| | 48|13.541127198108807| | 106|23.443430654272632| | 68| 30.56017232750517| | 13|24.744506938809245| | 20|21.409741525972343| +-----+------------------+ 30.459401036784325 18 +-----+------------------+ |count| prediction| +-----+------------------+ | 14| 24.09021006223882| | 13|22.438238079994537| | 16|22.438238079994537| | 21|22.438238079994537| | 51|22.438238079994537| | 163| 23.0366046202864| | 108| 23.0366046202864| | 33|22.438238079994537| | 21|23.896364953130924| | 16|29.393555023899953| | 5| 24.76302184836721| | 4| 24.76302184836721| +-----+------------------+ 49.14241559825254 19 +-----+------------------+ |count| prediction| +-----+------------------+ | 3|20.256010932200194| | 2| 22.03718853046706| | 0|20.256010932200194| | 1|20.256010932200194| | 0|20.256010932200194| | 7|20.256010932200194| | 22|20.766826443328394| | 59| 20.28388493834195| | 105| 19.77306942721375| | 100|18.997178828836933| | 38|23.713182849262132| | 14|19.362488170757803| +-----+------------------+ 38.32405795410877 20 +-----+------------------+ |count| prediction| +-----+------------------+ | 22|23.721652551989465| | 21| 26.6728443662679| | 20| 24.87133160594182| | 19|31.560198426666766| | 41|24.114729763373422| | 147|23.729370146693174| | 123|31.174838809986518| | 33|31.174838809986518| | 20| 32.32451786393887| | 16|27.311973630035613| | 4|27.311973630035613| | 3|24.114729763373422| +-----+------------------+ 46.00880359477889 21 +-----+------------------+ |count| prediction| +-----+------------------+ | 3| 16.80867227553702| | 3|20.302567049622592| | 0|19.884021077918234| | 1|19.803612968482543| | 1| 16.80867227553702| | 6|19.803612968482543| | 16|18.525819106227686| | 56| 23.13666059632795| | 83| 26.63055537041352| | 62|24.892265316555367| | 18|24.892265316555367| | 10| 19.2171123370851| +-----+------------------+ 24.921205566488197 22 +-----+-------------------+ |count| prediction| +-----+-------------------+ | 14| 8.984006285004082| | 12| 9.412254148030385| | 18|-1.0441821199065564| | 4| 52.939832871418965| | 30| 8.136211523629036| | 79| 10.30226506919652| | 76|0.18351010956678815| | 7| 12.116249745824959| | 5| 58.235952970408206| | 3| 18.36056965879067| | 3| 14.464932060681896| | 1| 53.97221415730735| +-----+-------------------+ 40.58825655938536 23 +-----+------------------+ |count| prediction| +-----+------------------+ | 5|15.415188515273485| | 1|11.968104030390489| | 1|20.263657898687903| | 0|12.869105157386103| | 0|14.636168205017112| | 3|13.677895690040518| | 21| 12.96568069173205| | 52| 9.696909643729965| | 75| 47.01941563924919| | 67|51.802006541554675| | 20| 50.90174653554755| | 11|57.674898157888244| +-----+------------------+ 24.315925201533894 24 +-----+------------------+ |count| prediction| +-----+------------------+ | 19| 77.85514834247051| | 16| 37.09861947248614| | 25|41.950204917728044| | 31| 48.68857950384889| | 52| 50.0210473080312| | 128| 78.62607468403225| | 67|25.400940589652663| | 35|21.604687096345646| | 21|27.602702257275794| | 8|26.785822037123175| | 11|26.785822037123175| | 8|26.785822037123175| +-----+------------------+ 28.663993821894177 25 +-----+------------------+ |count| prediction| +-----+------------------+ | 3|28.333499672434424| | 2|15.220584112258322| | 3|17.894003874392624| | 2|17.894003874392624| | 0|18.192058529131256| | 0|18.192058529131256| | 3|18.604127761322747| | 6|18.695998627741574| | 6|18.695998627741574| | 6|29.930839985709454| | 5| 32.15630812190821| | 16|26.115094176144808| +-----+------------------+ 18.099847182770354 26 +-----+------------------+ |count| prediction| +-----+------------------+ | 9| 23.75705474722169| | 19|29.418592365468907| | 18|17.426175548416712| | 23|17.426175548416712| | 21|14.453211122478104| | 23|22.226347191879128| | 25|21.162216885845975| | 18|21.162216885845975| | 17|20.283125531798653| | 14| 22.09221627703422| | 14|14.285106708700011| | 13|12.428438969021075| +-----+------------------+ 6.471556669552194 27 +-----+------------------+ |count| prediction| +-----+------------------+ | 4|15.665911282961106| | 3| 19.38598955366075| | 1|15.665911282961106| | 0| 19.38598955366075| | 0| 18.25993603329245| | 3| 18.25993603329245| | 4| 19.38598955366075| | 4|20.238445668853764| | 5| 18.74255177933975| | 3| 17.53087415532175| | 13|30.354347649438946| | 13| 36.35037171153647| +-----+------------------+ 16.604777582934908 28 +-----+------------------+ |count| prediction| +-----+------------------+ | 15| 32.49493422431936| | 18|24.397728223122858| | 33| 21.71779143797959| | 20| 24.07200377121302| | 26| 32.33212326274208| | 16|39.354789099996424| | 8|169.99459813648383| | 9| 171.0654512666404| | 7|190.13647584365646| | 3|167.12777182079617| | 4|21.847749932088565| | 2|30.105603521859276| +-----+------------------+ 97.96650650754756 29 +-----+------------------+ |count| prediction| +-----+------------------+ | 4| 21.3420441458323| | 1| 21.3420441458323| | 0| 13.66016139140512| | 0| 13.66016139140512| | 0| 13.66016139140512| | 7| 15.07925101280391| | 28| 13.66016139140512| | 68| 13.66016139140512| | 92| 21.3420441458323| | 66| 21.3420441458323| | 18|28.220531376291955| | 10| 20.78521741788431| +-----+------------------+ 31.232633219770282 30 +-----+------------------+ |count| prediction| +-----+------------------+ | 11| 24.86344297823159| | 18|24.673937734221216| | 14|24.673937734221216| | 22| 44.70559408361369| | 54| 77.34250916279265| | 158|44.774449220871276| | 109|30.278432251798428| | 43| 30.20957711454084| | 21|26.961455277888174| | 8|25.077156797946493| | 8| 24.07540224781928| | 4|25.077156797946493| +-----+------------------+ 42.4476393184953 31 +-----+------------------+ |count| prediction| +-----+------------------+ | 1|26.113278117704787| | 5|26.113278117704787| | 1| 19.44878323698| | 0|23.878584489124492| | 0| 19.44878323698| | 3|34.888743389756165| | 25|18.551076614157033| | 51|30.306395653548208| | 92|33.256641511354665| | 76|38.248297845974754| | 23|17.891290254682694| | 20|21.906108662504963| +-----+------------------+ 27.01490020406902 32 +-----+------------------+ |count| prediction| +-----+------------------+ | 16|15.097568573314803| | 10|14.890960821187655| | 22| 17.71888968152936| | 19|53.681598968755665| | 53| 69.04375289092079| | 161| 73.93074699302801| | 105| 69.04375289092079| | 37| 63.16588123240809| | 21| 63.16588123240809| | 20| 73.93074699302801| | 5|40.770237644457424| | 3| 50.75125767040706| +-----+------------------+ 40.112361443881994 33 +-----+-------------------+ |count| prediction| +-----+-------------------+ | 2| -8.461823304082023| | 0|-17.261627833432065| | 1| 11.227767333352627| | 2| 19.276493884953073| | 1| 11.227767333352627| | 8| 10.211086683279763| | 19| 6.540971982646021| | 63| -12.11779087753913| | 83|-5.0232441112375055| | 62| 52.9543906063527| | 25| 35.844101421175424| | 8| 33.32237331177195| +-----+-------------------+ 35.714884503564335 34 +-----+------------------+ |count| prediction| +-----+------------------+ | 13| 21.63952838338922| | 15|18.008990739783652| | 15| 41.4432317464839| | 16|46.901256324661894| | 39|29.484767262166674| | 145| 33.98960204807983| | 110| 32.25371806179067| | 41| 35.71014773004641| | 7|0.2950236508234202| | 5| 63.47522164468109| | 3|10.612833057045894| | 3|1.5481397951419447| +-----+------------------+ 44.48643426446519 35 +-----+------------------+ |count| prediction| +-----+------------------+ | 2|16.705383161247706| | 1|16.492736704917846| | 0|3.1132566514570508| | 0| 4.768375493117132| | 0| 24.57404727368431| | 3|13.455914942171239| | 14|13.987263707766147| | 47|15.160688517285143| | 43|21.931200654280172| | 28|3.3259031077869095| | 7| 5.243816957269909| | 9|22.345688813763203| +-----+------------------+ 16.954723488976427 36 +-----+------------------+ |count| prediction| +-----+------------------+ | 9|19.163821855191266| | 12| 12.17315685742773| | 30|15.631143156118059| | 19|15.848931349253883| | 34|11.915344454375198| | 155|14.218667517313827| | 93|10.532613429662584| | 37| 10.78101017201807| | 15|10.532613429662584| | 6|10.532613429662584| | 5|12.129322608228156| | 5|14.144656037159493| +-----+------------------+ 48.554180649314816 37 +-----+------------------+ |count| prediction| +-----+------------------+ | 2| 9.924667793193157| | 5| 8.61683043272282| | 0| 8.61683043272282| | 0| 9.785651116368067| | 0| 10.24354873923144| | 6|11.464738840771801| | 14|11.464738840771801| | 40|12.525002906811029| | 58|12.480039889729175| | 59|17.195749324993272| | 15|15.132411941430528| | 17|15.132411941430528| +-----+------------------+ 20.34116268656242 38 +-----+------------------+ |count| prediction| +-----+------------------+ | 7|17.725242051759945| | 24|27.593671303686424| | 31| 66.65078994916074| | 43| 79.03075683081322| | 72| 79.03075683081322| | 91| 68.95186495387749| | 51|32.397168000083234| | 30| 35.90662624267324| | 17|32.397168000083234| | 8| 35.90662624267324| | 2| 79.03075683081322| | 8| 79.03075683081322| +-----+------------------+ 36.06545372592152 39 +-----+------------------+ |count| prediction| +-----+------------------+ | 6|24.006820441686795| | 1| 46.81428320563627| | 2| 39.57111714738873| | 0| 36.89695463939994| | 1| 36.89695463939994| | 1|44.105219191588915| | 1| 62.08734677190399| | 2| 62.08734677190399| | 4| 46.81428320563627| | 10|24.006820441686795| | 12|32.030884118434244| | 11| 26.10789521476662| +-----+------------------+ 39.09785945416394 40 +-----+------------------+ |count| prediction| +-----+------------------+ | 11| 70.02455924576694| | 12| 50.78493579220319| | 10| 50.78493579220319| | 15| 65.73287025717853| | 10|53.674871023732464| | 17| 67.94645452274278| | 10|53.674871023732464| | 13| 67.94645452274278| | 11| 65.73287025717853| | 9|58.617168558511466| | 4| 65.73287025717853| | 6| 64.75843838397275| +-----+------------------+ 51.133892556267526 41 +-----+------------------+ |count| prediction| +-----+------------------+ | 1|4.7165423842459155| | 2|13.338253040784453| | 0| 13.07261749930427| | 0|13.338253040784453| | 0| 9.915710620518581| | 1|22.956112919623997| | 1|13.338253040784453| | 4|12.431777014324659| | 5|54.574304797783086| | 3|17.064025643810538| | 5| 8.222553249575235| | 7| 81.93814625596065| +-----+------------------+ 28.24949805211898 42 +-----+------------------+ |count| prediction| +-----+------------------+ | 16|10.643192983214792| | 16|15.399957007953724| | 16| 27.25754376214232| | 15| 27.25754376214232| | 11|61.047643600726666| | 15| 24.19628826999038| | 16|16.131369495003916| | 6|10.362970537655755| | 7| 8.715798405189002| | 7| 43.08008904364292| | 3| 41.90482978082634| | 2|19.061571198739024| +-----+------------------+ 22.4050329223903 43 +-----+------------------+ |count| prediction| +-----+------------------+ | 3| 13.49831775154648| | 2|18.221049244558852| | 3|14.498983587659742| | 0|14.498983587659742| | 0|19.922902901323894| | 4|19.922902901323894| | 23|14.364532258791566| | 53|16.056917737031338| | 70|18.774212318865313| | 55|27.421806032579354| | 16|26.228658810257436| | 9| 8.559737679767618| +-----+------------------+ 22.903902847685266 44 +-----+-------------------+ |count| prediction| +-----+-------------------+ | 9| 12.623980165297931| | 14| 15.965393494904168| | 12| 28.466751164498632| | 14| 6.042019957420359| | 27|-13.535995643280058| | 89| 102.363807158359| | 15| 10.25581261259141| | 4| 83.90025518374071| | 9| 9.804248438109669| | 13| 45.14832201874119| | 8| 44.646916590394646| | 6| 12.373465930759883| +-----+-------------------+ 30.27316565978336 45 +-----+-------------------+ |count| prediction| +-----+-------------------+ | 0| 11.853576085841517| | 1| 47.79734581465977| | 0|-0.9639296011587255| | 1|-0.7685593458159544| | 0| 4.6913320612818366| | 1| 79.1062642148555| | 0| 4.416717320172759| | 10| 4.221347064829987| | 77| 14.023974346652029| | 63| 16.02276338585752| | 15| 17.328239393326385| | 17| 18.518772708848946| +-----+-------------------+ 34.98883282289614 46 +-----+------------------+ |count| prediction| +-----+------------------+ | 16|22.489559354147428| | 13| 18.16934520066622| | 18|20.008272228001797| | 22|18.313057708328213| | 31|21.325353845987767| | 116|16.744646464223727| | 138| 18.16934520066622| | 41|16.995976090342243| | 20|17.918015574547706| | 8| 18.16934520066622| | 11| 18.16934520066622| | 6| 18.16934520066622| +-----+------------------+ 45.89345351155377 47 +-----+------------------+ |count| prediction| +-----+------------------+ | 1| 22.03758309964643| | 0| 5.257660319723647| | 1| 22.03758309964643| | 1| 22.03758309964643| | 0| 22.03758309964643| | 6|24.524001235768623| | 26|10.041855610690485| | 62| 16.65522939338268| | 90| 22.13120578562024| | 93| 28.57271707507125| | 31| 22.76805335617768| | 22| 16.29294907203198| +-----+------------------+ 33.35712114561442 48 +-----+------------------+ |count| prediction| +-----+------------------+ | 28| 17.77364330595932| | 26|20.631205072684615| | 36| 28.4729103277732| | 36|22.254887948428365| | 54|20.782651061048615| | 118| 18.67912552905151| | 110|18.696439316146915| | 57|22.276748358351814| | 22|22.276748358351814| | 16|18.090542901924028| | 6| 19.62072178417933| | 6|17.668091257215455| +-----+------------------+ 42.04677520686724 49 +-----+------------------+ |count| prediction| +-----+------------------+ | 3| 13.7439271454672| | 6|15.371955018534408| | 2|13.364591450545284| | 2|17.672247133270492| | 2|16.428882447891354| | 6|16.428882447891354| | 21| 23.02748796086793| | 54|18.204943581617478| | 89|30.977154422328688| | 71| 31.93754917168908| | 24|20.269843467063918| | 13| 20.65326608931397| +-----+------------------+ 24.401847629658526 50 +-----+------------------+ |count| prediction| +-----+------------------+ | 13|21.209899583900807| | 23|16.026536317156275| | 13| 33.78957278667738| | 30| 26.19608242690129| | 55|22.528475246165772| | 142|16.026536317156275| | 112| 26.96804019988156| | 50| 34.65547372972327| | 27| 32.17546859095647| | 21| 32.48922068270568| | 22| 32.48922068270568| | 13| 17.50042547059527| +-----+------------------+ 45.863361525884436 51 +-----+------------------+ |count| prediction| +-----+------------------+ | 8| 33.00715315211512| | 0|24.552081921265028| | 1|24.992432103568152| | 1|18.177685496342068| | 2|18.654271874937205| | 7|16.943423504956854| | 23|16.943423504956854| | 37|21.963984915644314| | 71|27.906563456663257| | 67| 39.16726998886576| | 18| 29.67145339360266| | 12| 34.47462384884114| +-----+------------------+ 22.380998447885755 52 +-----+------------------+ |count| prediction| +-----+------------------+ | 18|28.823362374879675| | 22| 25.75748181137665| | 37|28.009910082756683| | 42| 9.800595783655854| | 80| 9.800595783655854| | 105|28.009910082756683| | 65|35.622026591645415| | 29|31.629330597781404| | 15| 30.30576025872656| | 23|26.788679265152496| | 11|26.788679265152496| | 7|24.638954712532794| +-----+------------------+ 33.89116366399727 53 +-----+------------------+ |count| prediction| +-----+------------------+ | 7| 23.27250659656469| | 2| 35.53697766932646| | 2|19.480568343214966| | 1|20.933700352333084| | 0| 29.46091579708931| | 0| 36.519386920981| | 5|30.850187124026455| | 4| 42.74328249639363| | 8|39.174966244435225| | 12|31.865028703856602| | 10|52.204770044911946| | 21| 36.45431968837311| +-----+------------------+ 28.65005499182864 54 +-----+------------------+ |count| prediction| +-----+------------------+ | 18|43.072072837039876| | 13|27.746038740736854| | 13| 21.53527962672211| | 23| 35.7354749643796| | 25|37.098053700416074| | 21|23.393139636097498| | 25|24.561923351789485| | 15|29.819378775533323| | 11|26.308142978904044| | 11|23.393139636097498| | 11|23.393139636097498| | 3|23.393139636097498| +-----+------------------+ 14.172723040585831 55 +-----+------------------+ |count| prediction| +-----+------------------+ | 2|20.427965414986986| | 1| 21.17788034214797| | 2|24.160659650879612| | 0|18.781294260756358| | 0|18.940117307768315| | 1|27.890988292950613| | 1|27.110470518258325| | 0|29.202537398055394| | 4|38.366391575339065| | 6|28.610088869733026| | 10|48.365316806672354| | 10| 42.8027645675872| +-----+------------------+ 26.531664504327317 56 +-----+------------------+ |count| prediction| +-----+------------------+ | 13|28.689478418638227| | 14| 25.66187233410415| | 21| 25.66187233410415| | 5|23.886766917960756| | 0|23.471558349610813| | 0|16.729877910620456| | 0|27.923932003496986| | 0| 25.66187233410415| | 0| 25.66187233410415| | 0|22.989597994437695| | 0|23.271878307481252| | 0| 8.446404554029758| +-----+------------------+ 20.06169220935807 57 +-----+------------------+ |count| prediction| +-----+------------------+ | 0|17.731119540354285| | 0|11.934976630249048| | 2| 16.04564928209186| | 1|11.016758580820744| | 0|11.016758580820744| | 6|11.016758580820744| | 14|13.092901022212162| | 44|13.999824899477238| | 92| 8.698079070551664| | 55| 11.43666438171431| | 23|10.908948245735894| | 10|28.355208614106303| +-----+------------------+ 30.443805624685886 58 +-----+------------------+ |count| prediction| +-----+------------------+ | 19|16.810435055556397| | 10| 14.44218606847635| | 16|17.014796085706312| | 19|18.987517289177507| | 47|18.226090347164114| | 148|17.437798928024865| | 135|22.777680335151565| | 41| 31.44808336362216| | 25| 21.87749389118125| | 5|23.583764166381034| | 6|47.487743975063815| | 3|23.400589491033493| +-----+------------------+ 52.50205935103569 59 +-----+------------------+ |count| prediction| +-----+------------------+ | 1| 36.36564244287403| | 1| 22.0335108644563| | 1| 42.10819464004774| | 0|21.892830941642284| | 0|21.549871467702285| | 8| 17.68859546953034| | 24|14.294141915634514| | 46| 20.79121125736992| | 94|15.120464739690412| | 47| 22.11415552670455| | 19|37.679382371896565| | 19| 48.71945754850123| +-----+------------------+ 33.19603557173788 60 +-----+------------------+ |count| prediction| +-----+------------------+ | 11|34.118223812026635| | 11| 31.87967667791037| | 25|26.541254575701057| | 11|19.263069344747134| | 42| 23.83912041476887| | 158| 25.80562985125877| | 113| 28.2282689605956| | 34| 28.2282689605956| | 23| 28.2282689605956| | 8|30.997442093313456| | 10| 25.0680339530201| | 5|25.844183298025374| +-----+------------------+ 47.68257720220505 61 +-----+------------------+ |count| prediction| +-----+------------------+ | 0| 24.55782268267621| | 1|18.679868121218988| | 1|21.483616704089965| | 3|22.602403304856715| | 0|21.483616704089965| | 6|22.602403304856715| | 24|22.602403304856715| | 61|17.330501809810468| | 103|29.671106620703974| | 64|29.671106620703974| | 20| 33.36515303635377| | 12| 23.41681572041636| +-----+------------------+ 30.59170858930127 <class 'list'> 732 [Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.18408527535736915, DEWP=0.06939409923243636, SLP=-0.411391642080337, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), prediction=61.59404460991423), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.14219939028218284, SLP=-0.3492867754665158, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), prediction=7.778935354101069), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.14219939028218284, SLP=-0.2871819088527123, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), prediction=15.966927358090784), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.06939409923243636, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=0, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=82.31708145370767), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.06939409923243636, SLP=-0.24060325889236406, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), prediction=82.31708145370767), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.1451290030187449, DEWP=0.14219939028218284, SLP=-0.20955082558545346, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=8, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), prediction=18.014751874827045), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.2150046813319293, SLP=-0.19402460893199816, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=16, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), prediction=11.58732212625986), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.2150046813319293, SLP=-0.14744595897164992, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=52, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), prediction=9.357385562682786), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.36061526343142225, SLP=-0.13191974231819462, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=69, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), prediction=12.637939746904813), Row(SPD=1.9506756559862992, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.28780997238167577, SLP=-0.08534109235784636, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=54, va_features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), prediction=25.24644542515382), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=0.36061526343142225, SLP=-0.10086730901130166, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=21, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), prediction=11.942369854237393), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.16297217562510521, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), prediction=12.256402693047209), Row(SPD=1.3348834165148888, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.2150046813319293, SLP=-0.1784983922785605, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), prediction=23.153034885322995), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.2150046813319293, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=17, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=17.922527217444244), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.2150046813319293, SLP=-0.3337605588130782, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=12, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), prediction=17.922527217444244), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.14219939028218284, SLP=-0.3182343421596229, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=27, va_features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), prediction=35.94808745540508), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.06939409923243636, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=47, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), prediction=15.677677873198391), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.06939409923243636, SLP=-0.44244407538722996, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=165, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), prediction=15.677677873198391), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.47349650869414056, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=128, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), prediction=18.427032192739624), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.06939409923243636, SLP=-0.4579702920406853, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=36, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), prediction=15.677677873198391)] +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+ | SPD| GUS|CLR|SCT|BKN|OVC|OBS|POB| VSB| TEMP| DEWP| SLP| ALT| STP| PCP01|count| va_features| features| prediction| +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+ |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.18408527535736915|0.06939409923243636| -0.411391642080337| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313| 2|[-1.1282855413707...|[-1.1282855413707...| 61.59404460991423| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.10178170576334064|0.14219939028218284| -0.3492867754665158|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313| 1|[-0.2045971821636...|[-0.2045971821636...| 7.778935354101069| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.01947813616931213|0.14219939028218284| -0.2871819088527123| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313| 1|[0.71909117704347...|[0.71909117704347...|15.966927358090784| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.01947813616931213|0.06939409923243636|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 0|[-1.1282855413707...|[-1.1282855413707...| 82.31708145370767| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.06282543342471639|0.06939409923243636|-0.24060325889236406|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 1|[-0.2045971821636...|[-0.2045971821636...| 82.31708145370767| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| -0.1451290030187449|0.14219939028218284|-0.20955082558545346|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 8|[0.71909117704347...|[0.71909117704347...|18.014751874827045| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.06282543342471639| 0.2150046813319293|-0.19402460893199816|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313| 16|[-0.2045971821636...|[-0.2045971821636...| 11.58732212625986| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.10178170576334064| 0.2150046813319293|-0.14744595897164992|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 52|[-0.2045971821636...|[-0.2045971821636...| 9.357385562682786| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.3486924145454262|0.36061526343142225|-0.13191974231819462|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 69|[-1.1282855413707...|[-1.1282855413707...|12.637939746904813| | 1.9506756559862992|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.5132995537334832|0.28780997238167577|-0.08534109235784636|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313| 54|[1.95067565598629...|[1.95067565598629...| 25.24644542515382| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.6779066929215403|0.36061526343142225|-0.10086730901130166|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313| 21|[0.41119505730777...|[0.41119505730777...|11.942369854237393| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.28780997238167577|-0.16297217562510521|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 15|[0.41119505730777...|[0.41119505730777...|12.256402693047209| | 1.3348834165148888|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.089424540891683| 0.2150046813319293| -0.1784983922785605|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313| 15|[1.33488341651488...|[1.33488341651488...|23.153034885322995| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684| 0.2150046813319293|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 17|[-0.2045971821636...|[-0.2045971821636...|17.922527217444244| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.25403168007974| 0.2150046813319293| -0.3337605588130782|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313| 12|[0.41119505730777...|[0.41119505730777...|17.922527217444244| |-0.2045971821636372|-0.3045476160907193| 0| 1| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684|0.14219939028218284| -0.3182343421596229| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313| 27|[-0.2045971821636...|[-0.2045971821636...| 35.94808745540508| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684|0.06939409923243636|-0.39586542542688175| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313| 47|[-0.2045971821636...|[-0.2045971821636...|15.677677873198391| | 1.0269872967791835|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.25403168007974|0.06939409923243636|-0.44244407538722996|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 165|[1.02698729677918...|[1.02698729677918...|15.677677873198391| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.28780997238167577|-0.47349650869414056|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 128|[0.71909117704347...|[0.71909117704347...|18.427032192739624| | 1.0269872967791835|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.06939409923243636| -0.4579702920406853|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 36|[1.02698729677918...|[1.02698729677918...|15.677677873198391| +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+ only showing top 20 rows
df_gbtr = july_prediction2_df.select("*").toPandas()
df_gbtr.to_csv('df_gbtr.csv')
df_gbtr
| SPD | GUS | CLR | SCT | BKN | OVC | OBS | POB | VSB | TEMP | DEWP | SLP | ALT | STP | PCP01 | count | va_features | features | prediction | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.184085 | 0.069394 | -0.411392 | -0.400542 | -0.402984 | -0.178119 | 2 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 61.594045 |
| 1 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.101782 | 0.142199 | -0.349287 | -0.347805 | -0.340400 | -0.178119 | 1 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 7.778935 |
| 2 | 0.719091 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.019478 | 0.142199 | -0.287182 | -0.295068 | -0.293462 | -0.178119 | 1 | [0.7190911770434785, -0.3045476160907193, 1.0,... | [0.7190911770434785, -0.3045476160907193, 1.0,... | 15.966927 |
| 3 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.019478 | 0.069394 | -0.256129 | -0.242331 | -0.246525 | -0.178119 | 0 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 82.317081 |
| 4 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | -0.062825 | 0.069394 | -0.240603 | -0.242331 | -0.246525 | -0.178119 | 1 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 82.317081 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 727 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.097945 | 0.924817 | 1.161473 | 0.302814 | 0.337777 | 0.332374 | -0.178119 | 61 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 17.330502 |
| 728 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | -0.349287 | 1.089425 | 1.161473 | 0.333867 | 0.337777 | 0.332374 | -0.178119 | 103 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 29.671107 |
| 729 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | -0.349287 | 1.171728 | 1.161473 | 0.333867 | 0.337777 | 0.332374 | -0.178119 | 64 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 29.671107 |
| 730 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | -0.349287 | 1.418639 | 1.161473 | 0.380445 | 0.390514 | 0.394957 | -0.178119 | 20 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 33.365153 |
| 731 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 1.418639 | 1.088668 | 0.364919 | 0.390514 | 0.394957 | -0.178119 | 12 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 23.416816 |
732 rows × 19 columns
#3.build the pipline of randomforest regressor
from pyspark.ml.regression import RandomForestRegressor
rf = RandomForestRegressor(numTrees=10, maxDepth=5, seed=101, featuresCol="features",labelCol='count')
rf_pipeline = Pipeline(stages=[vector, vIndexer, rf])
rf_model = rf_pipeline.fit(train_df)
rf_prediction = rf_model.transform(test_df)
rf_prediction.select("count", "va_features","features","prediction").show()
rf_rmse = evaluator.evaluate(rf_prediction)
rf_rmse
+-----+--------------------+--------------------+------------------+ |count| va_features| features| prediction| +-----+--------------------+--------------------+------------------+ | 5|[-0.2045971821636...|[-0.2045971821636...| 21.47971511161542| | 8|[0.41119505730777...|[0.41119505730777...| 17.02752736889692| | 2|[0.41119505730777...|[0.41119505730777...| 22.58416181643424| | 1|[-0.2045971821636...|[-0.2045971821636...| 22.58416181643424| | 0|[-1.1282855413707...|[-1.1282855413707...| 15.90908989152083| | 1|[0.41119505730777...|[0.41119505730777...| 15.90908989152083| | 5|[-1.1282855413707...|[-1.1282855413707...| 15.90908989152083| | 1|[-1.1282855413707...|[-1.1282855413707...|18.188117222794812| | 10|[0.71909117704347...|[0.71909117704347...|19.605705286862126| | 11|[-1.1282855413707...|[-1.1282855413707...|18.356472050588092| | 13|[0.71909117704347...|[0.71909117704347...|20.422502371918718| | 20|[-1.1282855413707...|[-1.1282855413707...|20.111591250048935| | 22|[0.71909117704347...|[0.71909117704347...|19.516862400089632| | 20|[-1.1282855413707...|[-1.1282855413707...|20.593926512015933| | 28|[0.41119505730777...|[0.41119505730777...|29.915397462244506| | 19|[-0.2045971821636...|[-0.2045971821636...|29.915397462244506| | 9|[-0.2045971821636...|[-0.2045971821636...|26.781399677083932| | 21|[-0.2045971821636...|[-0.2045971821636...| 32.25658484374382| | 17|[-0.2045971821636...|[-0.2045971821636...|24.120640505794007| | 22|[0.71909117704347...|[0.71909117704347...| 32.58326276355699| +-----+--------------------+--------------------+------------------+ only showing top 20 rows
28.453972138637624
#realtime streaming of random forest fregressor
realtime_list=realtime_df.collect()
newtrain_list=train_df.collect()
july_prediction3_list=[]
for i in range(int(realtime_df.count()/12)):
print(i+1)
realtimepart_list=realtime_list[i*12:(i*12+12)]
realtimepart_df = spark.createDataFrame(realtimepart_list)
rf_prediction = rf_model.transform(realtimepart_df)
rf_prediction.select("count", "prediction").show()
rmse = evaluator.evaluate(rf_prediction)
print(rmse)
july_prediction3_list=july_prediction3_list+rf_prediction.collect()
newtrain_list=newtrain_list + realtimepart_list
newtrain_df = spark.createDataFrame(newtrain_list)
rf_model = rf_pipeline.fit(newtrain_df)
print(type(july_prediction3_list))
print(len(july_prediction3_list))
print(july_prediction3_list[0:20])
july_prediction3_df = spark.createDataFrame(july_prediction3_list)
july_prediction3_df.show()
1 +-----+------------------+ |count| prediction| +-----+------------------+ | 2| 17.95700665994376| | 1| 17.28247725096573| | 1| 18.70006531503304| | 0|18.188117222794812| | 1|18.188117222794812| | 8|19.520817025119957| | 16| 17.28247725096573| | 52| 17.28247725096573| | 69|20.111591250048935| | 54|22.180979483401217| | 21|20.111591250048935| | 15|20.111591250048935| +-----+------------------+ 22.782304886893844 2 +-----+------------------+ |count| prediction| +-----+------------------+ | 15|23.780114059765907| | 17|21.326904911897515| | 12|22.693524979302715| | 27|26.835392583369064| | 47|21.326904911897515| | 165|22.484917237808524| | 128|22.484917237808524| | 36|22.484917237808524| | 18|21.326904911897515| | 7|21.326904911897515| | 11|22.693524979302715| | 7|21.326904911897515| +-----+------------------+ 52.48354114310918 3 +-----+------------------+ |count| prediction| +-----+------------------+ | 2|23.161749925041597| | 3|21.830413519511644| | 1|23.161749925041597| | 2|27.924742110051888| | 0|25.205239412362978| | 5|21.830413519511644| | 23|21.830413519511644| | 52|21.830413519511644| | 102|23.161749925041597| | 66| 24.48024740655483| | 17| 24.48024740655483| | 14| 29.24323959156512| +-----+------------------+ 31.653657404847646 4 +-----+------------------+ |count| prediction| +-----+------------------+ | 17|27.879421552253245| | 15|26.534984052253247| | 19|26.534984052253247| | 25|30.246329494553926| | 44| 28.90189199455392| | 112| 28.22831667694956| | 122|30.246329494553926| | 28|29.582257723048855| | 14|29.572754176949566| | 13| 28.22831667694956| | 4|21.247351836035325| | 5|27.879421552253245| +-----+------------------+ 37.9670431519968 5 +-----+------------------+ |count| prediction| +-----+------------------+ | 1| 26.546641109065| | 1| 29.39209630848108| | 0|28.109720829553872| | 1| 34.76398564242925| | 0| 37.55731897576258| | 5| 41.98321858494312| | 17| 26.99969666462055| | 33| 26.99969666462055| | 58|35.402385251609786| | 39|26.825481234456852| | 17| 26.546641109065| | 13|26.825481234456852| +-----+------------------+ 24.57025091903441 6 +-----+------------------+ |count| prediction| +-----+------------------+ | 17| 26.57770885707053| | 37| 26.1666666734477| | 44| 26.57770885707053| | 78| 26.1666666734477| | 67|25.265842497623527| | 93| 31.06597868493288| | 36| 26.57770885707053| | 20| 26.57770885707053| | 15|22.786670988907836| | 7| 23.02980010673106| | 7|23.440842290353892| | 6| 20.4879840239389| +-----+------------------+ 28.441054138413175 7 +-----+------------------+ |count| prediction| +-----+------------------+ | 3| 20.2117554165253| | 0| 20.2117554165253| | 0| 20.2117554165253| | 0|28.451229100735826| | 0|15.592567683412991| | 0|16.671048696071217| | 2|16.671048696071217| | 3|15.592567683412991| | 2|18.970519186626362| | 5| 32.11269474430447| | 8|22.511225907080444| | 12|22.511225907080444| +-----+------------------+ 18.622946918934982 8 +-----+------------------+ |count| prediction| +-----+------------------+ | 14| 24.72920586002534| | 9| 24.72920586002534| | 18|27.846175590836975| | 19| 26.18101943651977| | 19| 32.43337252669201| | 16| 24.72920586002534| | 14| 24.72920586002534| | 12| 23.91393440348562| | 11| 19.61869724333341| | 3|22.019267043903213| | 15|14.088481737275302| | 7|14.088481737275302| +-----+------------------+ 11.22438607443106 9 +-----+------------------+ |count| prediction| +-----+------------------+ | 1|13.567562901886783| | 1|13.567562901886783| | 1|13.567562901886783| | 0|17.294540449899173| | 0|30.453132035400216| | 1|30.453132035400216| | 6| 21.67051246575619| | 15|30.453132035400216| | 26| 35.79923715173716| | 16|35.351737151737154| | 10| 40.05188144713975| | 8| 40.05188144713975| +-----+------------------+ 21.317305160587576 10 +-----+------------------+ |count| prediction| +-----+------------------+ | 15| 31.28141366480627| | 15|24.347247813627167| | 16|19.984475863355218| | 23| 41.60708329590126| | 24|26.092107661943505| | 49|16.601345372596736| | 21| 17.89896580771717| | 18| 29.25842162755442| | 15| 17.89896580771717| | 7| 17.89896580771717| | 7| 17.89896580771717| | 11| 17.89896580771717| +-----+------------------+ 13.535594800889863 11 +-----+------------------+ |count| prediction| +-----+------------------+ | 7| 17.07627712439396| | 2| 17.07627712439396| | 1| 17.07627712439396| | 1| 17.07627712439396| | 0|27.334862383310753| | 2|27.334862383310753| | 2|27.334862383310753| | 4|27.781529049977422| | 4|29.138009729791435| | 3|33.929342108792284| | 3| 42.59558548736719| | 15|33.957490249271956| +-----+------------------+ 24.07960021385361 12 +-----+------------------+ |count| prediction| +-----+------------------+ | 16| 23.02464323700992| | 14| 32.43283341225508| | 17| 30.50307150749317| | 12|23.950169268326782| | 17|23.950169268326782| | 12|29.206919780569557| | 18|29.795027178102238| | 14|24.797875612587887| | 13| 26.54940000283179| | 13| 26.54940000283179| | 8| 21.81702723102406| | 8| 21.81702723102406| +-----+------------------+ 13.112802696979612 13 +-----+------------------+ |count| prediction| +-----+------------------+ | 7| 21.76879202344602| | 3|24.838391115943704| | 1| 21.76879202344602| | 0|28.358648208908864| | 0| 21.76879202344602| | 0|27.619662190420883| | 1|24.630960917124543| | 3|23.559576606383537| | 4|23.559576606383537| | 5|28.754806861485935| | 6| 33.65462841177735| | 10|23.559576606383537| +-----+------------------+ 22.442410040058313 14 +-----+------------------+ |count| prediction| +-----+------------------+ | 13|18.245867589154123| | 17|21.595285095218273| | 19|20.195224523186898| | 10| 16.11260245228266| | 16| 19.91623070147463| | 20| 17.49765235624964| | 16|20.410005364904016| | 12|20.410005364904016| | 6|20.410005364904016| | 5|20.410005364904016| | 8|20.410005364904016| | 6| 24.21363361409599| +-----+------------------+ 9.70422755388818 15 +-----+------------------+ |count| prediction| +-----+------------------+ | 0|17.895940368144554| | 0|17.895940368144554| | 0| 21.73330479941315| | 0|22.474389528686856| | 0| 16.36773726125839| | 3|13.427768691567673| | 18|15.156509517987171| | 31| 16.36773726125839| | 51|11.168137887567338| | 32| 18.140879512793| | 30|17.895940368144554| | 3|16.817199541725053| +-----+------------------+ 19.00190679510246 16 +-----+------------------+ |count| prediction| +-----+------------------+ | 9|17.933899261835037| | 14|16.243725325161158| | 18| 17.22646892127776| | 20|16.243725325161158| | 52|16.243725325161158| | 165|16.243725325161158| | 118|16.243725325161158| | 38|16.243725325161158| | 19|16.243725325161158| | 12|16.243725325161158| | 7|16.243725325161158| | 3|16.243725325161158| +-----+------------------+ 53.71267784812599 17 +-----+------------------+ |count| prediction| +-----+------------------+ | 0| 17.33279456900212| | 1| 17.33279456900212| | 0| 18.28960897824345| | 0| 17.33279456900212| | 0| 16.74367382021152| | 7| 15.78685941097019| | 20| 15.78685941097019| | 48| 17.33279456900212| | 106| 18.28960897824345| | 68| 18.28960897824345| | 13|18.174492897269598| | 20|18.461567684168447| +-----+------------------+ 32.54648839671255 18 +-----+------------------+ |count| prediction| +-----+------------------+ | 14|22.039371597686443| | 13| 16.42593255059399| | 16| 16.42593255059399| | 21| 16.42593255059399| | 51| 16.42593255059399| | 163| 16.42593255059399| | 108| 16.42593255059399| | 33| 16.42593255059399| | 21| 16.42593255059399| | 16| 16.42593255059399| | 5| 16.42593255059399| | 4|15.348163324378849| +-----+------------------+ 51.41152692397727 19 +-----+------------------+ |count| prediction| +-----+------------------+ | 3| 16.16907433066968| | 2| 16.93563539513982| | 0|17.471539316664078| | 1|17.471539316664078| | 0| 16.16907433066968| | 7| 16.16907433066968| | 22| 16.16907433066968| | 59| 16.16907433066968| | 105| 16.16907433066968| | 100| 16.16907433066968| | 38| 22.17107905728583| | 14| 16.16907433066968| +-----+------------------+ 39.116151682193 20 +-----+------------------+ |count| prediction| +-----+------------------+ | 22|20.788109318197264| | 21|19.758060191762564| | 20| 20.39889088929084| | 19|22.466424863659892| | 41|18.241034577071964| | 147|19.042648005382638| | 123|19.042648005382638| | 33|19.042648005382638| | 20|19.758060191762564| | 16| 19.68347870291091| | 4| 19.68347870291091| | 3|18.241034577071964| +-----+------------------+ 48.64845547672396 21 +-----+------------------+ |count| prediction| +-----+------------------+ | 3|19.914905929234617| | 3|20.679428706848704| | 0|19.914905929234617| | 1| 20.47928391053606| | 1|19.914905929234617| | 6| 20.47928391053606| | 16|19.914905929234617| | 56|19.914905929234617| | 83|21.391357130643264| | 62|23.225932214308436| | 18|23.225932214308436| | 10|27.559935620800264| +-----+------------------+ 27.224867567180077 22 +-----+------------------+ |count| prediction| +-----+------------------+ | 14|22.921345582952068| | 12| 21.86261058631818| | 18|19.891382454916194| | 4| 7.647371393352563| | 30| 9.706656484266452| | 79|11.687201964323616| | 76| 11.8031621630797| | 7|10.692507175201847| | 5|15.608244880119878| | 3|10.692507175201847| | 3|12.430420107340131| | 1| 25.29739773689922| +-----+------------------+ 29.043715892973058 23 +-----+------------------+ |count| prediction| +-----+------------------+ | 5|31.952847474221624| | 1|32.834946237128435| | 1|31.972051932450874| | 0|32.834946237128435| | 0|31.952847474221624| | 3| 38.91723534283709| | 21| 33.85692639662724| | 52|31.952847474221624| | 75| 39.11804302597094| | 67|39.544031335680174| | 20| 39.71305727098518| | 11|39.544031335680174| +-----+------------------+ 28.723181730642832 24 +-----+------------------+ |count| prediction| +-----+------------------+ | 19| 43.4945762166063| | 16| 33.78786653619724| | 25| 31.72998314052315| | 31|39.924237003580515| | 52| 35.97629389004166| | 128| 35.68050013156134| | 67| 31.19478618426004| | 35| 27.71085766974151| | 21|24.059289737120075| | 8| 20.20076681647911| | 11| 20.20076681647911| | 8| 20.20076681647911| +-----+------------------+ 31.021299229177675 25 +-----+------------------+ |count| prediction| +-----+------------------+ | 3|21.993243009875435| | 2|22.360698841740316| | 3|20.336824428456847| | 2|20.663458320880217| | 0|19.799846509038698| | 0|19.799846509038698| | 3|19.799846509038698| | 6|19.473212616615328| | 6|19.473212616615328| | 6| 22.82687171889401| | 5|21.129631198033913| | 16|19.894180416930958| +-----+------------------+ 16.86224172863294 26 +-----+------------------+ |count| prediction| +-----+------------------+ | 9|21.917931957690445| | 19|30.688065894151208| | 18|21.917931957690445| | 23|21.917931957690445| | 21|23.009624030861175| | 23|27.081154312211133| | 25|25.989462239040403| | 18|25.989462239040403| | 17|25.989462239040403| | 14|25.989462239040403| | 14|21.917931957690445| | 13|18.495336699303152| +-----+------------------+ 7.761871940204926 27 +-----+------------------+ |count| prediction| +-----+------------------+ | 4|19.965663440096872| | 3|21.886280219563098| | 1|21.948991130699206| | 0|21.948991130699206| | 0| 20.02837435123298| | 3| 20.02837435123298| | 4| 20.02837435123298| | 4| 20.02837435123298| | 5|19.965663440096872| | 3| 21.61331034178032| | 13| 31.83742080767338| | 13|26.698174831498466| +-----+------------------+ 17.911928362664405 28 +-----+------------------+ |count| prediction| +-----+------------------+ | 15|32.144453558657304| | 18|23.153661794523863| | 33| 24.85748763580658| | 20|26.837487635806582| | 26|26.837487635806582| | 16| 46.29597325757062| | 8|58.035201647694876| | 9| 57.00551984869173| | 7| 36.0103457739402| | 3|27.131774345368775| | 4|20.567060629571916| | 2| 28.46014946176762| +-----+------------------+ 26.70626126014692 29 +-----+------------------+ |count| prediction| +-----+------------------+ | 4|19.896479210420246| | 1|19.896479210420246| | 0|18.206799638084142| | 0|18.206799638084142| | 0|18.206799638084142| | 7|17.418482042089963| | 28|18.206799638084142| | 68|18.206799638084142| | 92|19.896479210420246| | 66|19.896479210420246| | 18|19.896479210420246| | 10| 22.14807879642476| +-----+------------------+ 31.30984505974298 30 +-----+------------------+ |count| prediction| +-----+------------------+ | 11|21.868568243631536| | 18| 25.75312153550439| | 14| 25.75312153550439| | 22|31.072224969648158| | 54|25.568506150889007| | 158|31.072224969648158| | 109|21.868568243631536| | 43|21.868568243631536| | 21| 25.34135091176194| | 8| 21.59173060227451| | 8| 21.59173060227451| | 4|20.728094238638143| +-----+------------------+ 46.56917638640825 31 +-----+------------------+ |count| prediction| +-----+------------------+ | 1|19.363358875173116| | 5|19.363358875173116| | 1|18.587243120670582| | 0|19.363358875173116| | 0|18.587243120670582| | 3|18.587243120670582| | 25|18.587243120670582| | 51|19.363358875173116| | 92|19.363358875173116| | 76|21.048926962056992| | 23|24.983312814560044| | 20|23.947125651100873| +-----+------------------+ 30.514250051954182 32 +-----+------------------+ |count| prediction| +-----+------------------+ | 16| 24.06934109937149| | 10| 24.06934109937149| | 22|28.002046067257776| | 19|31.199787160773447| | 53| 40.96659600133227| | 161| 40.96659600133227| | 105| 40.96659600133227| | 37| 40.96659600133227| | 21| 40.96659600133227| | 20| 32.90659600133227| | 5|28.002046067257776| | 3|25.589749446481804| +-----+------------------+ 41.55457021281788 33 +-----+------------------+ |count| prediction| +-----+------------------+ | 2|24.414352823715976| | 0|24.414352823715976| | 1| 17.50700693628664| | 2| 17.50700693628664| | 1|18.331513586906052| | 8|18.937469715491957| | 19| 17.50700693628664| | 63|24.500670280722186| | 83| 21.32568613195813| | 62| 31.17009256502762| | 25| 31.77004534882723| | 8|21.459445075110192| +-----+------------------+ 26.609903798262998 34 +-----+------------------+ |count| prediction| +-----+------------------+ | 13| 20.83469994509859| | 15|22.606623550582178| | 15|22.606623550582178| | 16| 28.83103499369779| | 39|22.606623550582178| | 145| 31.69497478580727| | 110|35.267052707885185| | 41|35.267052707885185| | 7| 10.8049533153874| | 5|16.219372612879294| | 3| 9.659282895548156| | 3| 7.050669277828713| +-----+------------------+ 40.07024770489666 35 +-----+------------------+ |count| prediction| +-----+------------------+ | 2|15.633681012676925| | 1|15.633681012676925| | 0|11.940032410588929| | 0|25.757080828042064| | 0| 22.2595864239091| | 3| 17.85032289930304| | 14| 17.85032289930304| | 47|16.666690036658625| | 43|16.725393549717648| | 28|10.351833293043319| | 7|11.938428147994165| | 9| 22.2595864239091| +-----+------------------+ 18.391544387213806 36 +-----+------------------+ |count| prediction| +-----+------------------+ | 9|18.468463871984543| | 12|13.411697096588933| | 30|18.468463871984543| | 19|17.941721546268724| | 34|19.536415454852325| | 155| 18.23725980140152| | 93| 18.23725980140152| | 37|19.536415454852325| | 15| 18.23725980140152| | 6| 18.23725980140152| | 5|19.114640753782474| | 5|18.468463871984543| +-----+------------------+ 46.16481384523791 37 +-----+------------------+ |count| prediction| +-----+------------------+ | 2|17.281471537356932| | 5| 18.58554624024318| | 0| 18.58554624024318| | 0|18.658553035260166| | 0|17.354478332373922| | 6|17.354478332373922| | 14|17.354478332373922| | 40|18.658553035260166| | 58| 18.58554624024318| | 59|18.585567122607483| | 15|17.354478332373922| | 17|17.556296514192102| +-----+------------------+ 20.818951969382805 38 +-----+------------------+ |count| prediction| +-----+------------------+ | 7| 31.12172804552434| | 24|33.607053015942014| | 31| 36.75324469902248| | 43|36.876284081612425| | 72| 56.14429754962589| | 91| 51.13620766198544| | 51| 46.21781048052706| | 30| 37.0011438138604| | 17| 46.21781048052706| | 8| 37.0011438138604| | 2| 41.36829110840206| | 8| 41.36829110840206| +-----+------------------+ 24.178529614643175 39 +-----+------------------+ |count| prediction| +-----+------------------+ | 6| 31.77804720118821| | 1| 39.79377807997936| | 2| 39.79377807997936| | 0| 39.79377807997936| | 1| 39.79377807997936| | 1| 33.10203260510118| | 1| 39.79377807997936| | 2| 37.89156531402192| | 4| 37.89156531402192| | 10| 31.77804720118821| | 12| 36.73965514144863| | 11|30.007524813128505| +-----+------------------+ 33.04831922042469 40 +-----+------------------+ |count| prediction| +-----+------------------+ | 11|29.278555601664003| | 12|29.278555601664003| | 10| 30.79882104321581| | 15|30.341182434910536| | 10|31.364932154326926| | 17| 52.70232366366745| | 10| 30.79882104321581| | 13| 52.13621255255633| | 11|30.341182434910536| | 9|29.278555601664003| | 4|28.194118745911368| | 6| 30.79882104321581| +-----+------------------+ 24.120505292588813 41 +-----+------------------+ |count| prediction| +-----+------------------+ | 1| 25.41849531546238| | 2| 26.12180206398596| | 0|25.854213131179637| | 0| 26.12180206398596| | 0| 25.95111590582435| | 1|28.198174729353763| | 1|22.852812248184897| | 4|22.626873338545472| | 5| 35.81000463441768| | 3|23.981578098873506| | 5| 24.69467060950064| | 7| 35.89621207170452| +-----+------------------+ 24.797469955757144 42 +-----+------------------+ |count| prediction| +-----+------------------+ | 16|24.638419516964245| | 16| 22.4504804184753| | 16| 19.07708343209191| | 15|31.096080495952197| | 11|31.096080495952197| | 15| 33.33732206688127| | 16|27.318662236657122| | 6| 22.00401559677136| | 7|23.958302927401533| | 7|25.616084980791037| | 3|25.616084980791037| | 2|22.639605523067424| +-----+------------------+ 16.022788394683694 43 +-----+------------------+ |count| prediction| +-----+------------------+ | 3|20.624866490376515| | 2| 18.830466616899| | 3| 19.51312084374124| | 0| 19.51312084374124| | 0| 19.51312084374124| | 4| 19.51312084374124| | 23| 20.09409255793394| | 53| 20.09409255793394| | 70| 24.33942469585708| | 55|23.474040080472463| | 16|23.474040080472463| | 9| 20.53889975440978| +-----+------------------+ 22.78118592364291 44 +-----+------------------+ |count| prediction| +-----+------------------+ | 9|23.039572345792557| | 14|21.123749270158516| | 12|21.586753624529727| | 14|17.781152778804582| | 27|14.816445266419738| | 89|27.139402481614148| | 15| 11.39852909995315| | 4|15.795331878369353| | 9|15.737296598018935| | 13|18.827797985253717| | 8|22.770710725406097| | 6| 21.23573166828107| +-----+------------------+ 20.43483370856744 45 +-----+------------------+ |count| prediction| +-----+------------------+ | 0| 9.172699837440224| | 1|12.072032937386512| | 0| 9.818781863930813| | 1| 8.83965780735877| | 0| 7.895679890067337| | 1|10.391681268414075| | 0| 9.288392557658645| | 10| 6.948912038178125| | 77|25.973230919748865| | 63|22.952128157712565| | 15|22.952128157712565| | 17|20.121900453507312| +-----+------------------+ 20.18925387856117 46 +-----+------------------+ |count| prediction| +-----+------------------+ | 16|21.914689129581355| | 13|20.840905247250486| | 18|20.840905247250486| | 22| 20.18974549246884| | 31|28.272048575292057| | 116|19.613818575523517| | 138|20.840905247250486| | 41|18.487686096891036| | 20| 19.71477276861801| | 8|18.487686096891036| | 11|18.487686096891036| | 6|18.487686096891036| +-----+------------------+ 44.684717015539235 47 +-----+------------------+ |count| prediction| +-----+------------------+ | 1| 20.60289278065635| | 0|10.771993070947676| | 1|15.897065819754715| | 1|21.216740233513093| | 0| 20.60289278065635| | 6| 20.60289278065635| | 26| 20.60289278065635| | 62| 20.60289278065635| | 90| 20.60289278065635| | 93| 20.60289278065635| | 31| 20.60289278065635| | 22| 20.60289278065635| +-----+------------------+ 33.76249483279547 48 +-----+------------------+ |count| prediction| +-----+------------------+ | 28| 20.8339090256351| | 26| 25.7849651291788| | 36| 21.68352764085801| | 36| 21.06952174726245| | 54|21.919140362485358| | 118|22.428803803648705| | 110|21.579185188425797| | 57|21.579185188425797| | 22|21.579185188425797| | 16| 20.8339090256351| | 6| 20.8339090256351| | 6| 20.74708065434801| +-----+------------------+ 41.00432332819352 49 +-----+------------------+ |count| prediction| +-----+------------------+ | 3| 20.63056911190121| | 6| 20.63056911190121| | 2|19.920861314996824| | 2| 21.45490371837408| | 2|20.483909037997247| | 6|20.483909037997247| | 21|19.925827597972223| | 54|19.920861314996824| | 89| 20.63056911190121| | 71| 20.63056911190121| | 24| 20.63056911190121| | 13| 19.3627798749718| +-----+------------------+ 29.157357280678443 50 +-----+------------------+ |count| prediction| +-----+------------------+ | 13|22.233989579505526| | 23|21.511608671955162| | 13|20.308637462688132| | 30|21.483500957213625| | 55|22.304147237163185| | 142|20.690962392005602| | 112|19.100369263007938| | 50|19.670246312188265| | 27|19.670246312188265| | 21|19.670246312188265| | 22|19.670246312188265| | 13|19.670246312188265| +-----+------------------+ 46.23669090038388 51 +-----+------------------+ |count| prediction| +-----+------------------+ | 8|21.712050281705583| | 0|27.368322469030023| | 1|27.368322469030023| | 1|27.368322469030023| | 2|22.702069890340788| | 7|22.702069890340788| | 23|22.702069890340788| | 37|27.368322469030023| | 71|27.368322469030023| | 67|25.513998204486775| | 18| 29.09090042529192| | 12| 29.09090042529192| +-----+------------------+ 24.384984292062708 52 +-----+------------------+ |count| prediction| +-----+------------------+ | 18| 34.04332621280515| | 22| 29.4820876241371| | 37| 28.3277797825316| | 42|26.985921566287676| | 80|26.985921566287676| | 105| 28.3277797825316| | 65|28.090853799565952| | 29|23.905569178471502| | 15|24.754936291318824| | 23|25.369007275815555| | 11|25.369007275815555| | 7|22.924799947702272| +-----+------------------+ 30.621613650572503 53 +-----+------------------+ |count| prediction| +-----+------------------+ | 7| 28.07452746947276| | 2|29.654088735572753| | 2| 28.68109714969384| | 1|29.092053891174135| | 0| 28.07452746947276| | 0| 28.07452746947276| | 5| 28.07452746947276| | 4|33.930404134972186| | 8| 31.84437513098274| | 12|29.698623571395217| | 10| 36.00991424403212| | 21| 33.1304197041641| +-----+------------------+ 24.86632275215677 54 +-----+------------------+ |count| prediction| +-----+------------------+ | 18| 36.17682417328529| | 13|33.965268617729734| | 13|28.395117061389158| | 23|28.350771307263983| | 25|33.965268617729734| | 21|23.697831672982865| | 25|22.286863074746844| | 15|23.873619363553168| | 11| 25.01590559217603| | 11|23.697831672982865| | 11|23.697831672982865| | 3|23.697831672982865| +-----+------------------+ 13.406006370863338 55 +-----+------------------+ |count| prediction| +-----+------------------+ | 2|21.947450904015394| | 1|24.658762048583725| | 2|26.048159991375105| | 0|22.412838130282513| | 0|24.658762048583725| | 1|24.658762048583725| | 1|22.993532982875173| | 0|22.412838130282513| | 4|27.790645867081317| | 6|24.578892160350414| | 10|27.954859416568667| | 10|30.418192749902005| +-----+------------------+ 22.065393608823246 56 +-----+------------------+ |count| prediction| +-----+------------------+ | 13|25.296200910552567| | 14|22.853343767695424| | 21|22.853343767695424| | 5| 26.96609048688663| | 0| 26.96609048688663| | 0| 25.01338125985676| | 0| 26.96609048688663| | 0|24.273624892193414| | 0|24.273624892193414| | 0|25.428814288497016| | 0|23.593100135358764| | 0|13.975409910358763| +-----+------------------+ 21.15915949129792 57 +-----+------------------+ |count| prediction| +-----+------------------+ | 0|11.229757938840887| | 0|19.003721793491255| | 2| 19.77197598189801| | 1|19.003721793491255| | 0|19.003721793491255| | 6|19.003721793491255| | 14| 19.77197598189801| | 44|20.829686286119937| | 92|20.829686286119937| | 55|24.892231011774246| | 23| 26.03385927598315| | 10| 23.44270159147279| +-----+------------------+ 26.440570525668903 58 +-----+------------------+ |count| prediction| +-----+------------------+ | 19|26.192541558645406| | 10|22.351081424297604| | 16| 24.58704173159594| | 19|27.985351655209463| | 47| 23.55601377417769| | 148|22.351081424297604| | 135| 26.65363681507511| | 41|24.017109030607394| | 25| 23.55601377417769| | 5| 23.55601377417769| | 6|26.163019472157732| | 3| 23.55601377417769| +-----+------------------+ 49.91566865465443 59 +-----+------------------+ |count| prediction| +-----+------------------+ | 1|25.427586867849264| | 1| 25.32068422551108| | 1|25.427586867849264| | 0|23.285857633437704| | 0|21.094120938759833| | 8| 20.19868672834622| | 24|16.735529795859584| | 46|20.403374597440347| | 94|20.734997436734247| | 47|20.488333516165117| | 19|21.094120938759833| | 19|30.044300202961068| +-----+------------------+ 28.615062281573945 60 +-----+------------------+ |count| prediction| +-----+------------------+ | 11| 24.7375041951986| | 11|27.726757807004862| | 25| 24.74476628158113| | 11|19.371271982491365| | 42| 23.3966833272657| | 158|26.068227811018346| | 113|26.067805152790942| | 34| 26.97680180130372| | 23| 26.97680180130372| | 8|24.737926853426007| | 10|24.280897326525128| | 5|24.737926853426007| +-----+------------------+ 47.246479091026174 61 +-----+------------------+ |count| prediction| +-----+------------------+ | 0| 21.56314553102278| | 1|20.818676977580957| | 1|19.420193669996326| | 3| 20.36095178976571| | 0|19.420193669996326| | 6|21.453041204892788| | 24| 20.36095178976571| | 61| 19.63766517009006| | 103| 20.34608303102278| | 64| 20.34608303102278| | 20| 21.12524619070971| | 12| 22.37893056591924| +-----+------------------+ 32.51256685123153 <class 'list'> 732 [Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.18408527535736915, DEWP=0.06939409923243636, SLP=-0.411391642080337, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), prediction=17.95700665994376), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.14219939028218284, SLP=-0.3492867754665158, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), prediction=17.28247725096573), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.14219939028218284, SLP=-0.2871819088527123, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), prediction=18.70006531503304), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.06939409923243636, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=0, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=18.188117222794812), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.06939409923243636, SLP=-0.24060325889236406, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), prediction=18.188117222794812), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.1451290030187449, DEWP=0.14219939028218284, SLP=-0.20955082558545346, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=8, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), prediction=19.520817025119957), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.2150046813319293, SLP=-0.19402460893199816, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=16, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), prediction=17.28247725096573), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.2150046813319293, SLP=-0.14744595897164992, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=52, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), prediction=17.28247725096573), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.36061526343142225, SLP=-0.13191974231819462, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=69, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), prediction=20.111591250048935), Row(SPD=1.9506756559862992, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.28780997238167577, SLP=-0.08534109235784636, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=54, va_features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), prediction=22.180979483401217), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=0.36061526343142225, SLP=-0.10086730901130166, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=21, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), prediction=20.111591250048935), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.16297217562510521, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), prediction=20.111591250048935), Row(SPD=1.3348834165148888, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.2150046813319293, SLP=-0.1784983922785605, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), prediction=23.780114059765907), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.2150046813319293, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=17, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=21.326904911897515), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.2150046813319293, SLP=-0.3337605588130782, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=12, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), prediction=22.693524979302715), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.14219939028218284, SLP=-0.3182343421596229, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=27, va_features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), prediction=26.835392583369064), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.06939409923243636, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=47, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), prediction=21.326904911897515), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.06939409923243636, SLP=-0.44244407538722996, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=165, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), prediction=22.484917237808524), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.47349650869414056, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=128, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), prediction=22.484917237808524), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.06939409923243636, SLP=-0.4579702920406853, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=36, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), prediction=22.484917237808524)] +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+ | SPD| GUS|CLR|SCT|BKN|OVC|OBS|POB| VSB| TEMP| DEWP| SLP| ALT| STP| PCP01|count| va_features| features| prediction| +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+ |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.18408527535736915|0.06939409923243636| -0.411391642080337| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313| 2|[-1.1282855413707...|[-1.1282855413707...| 17.95700665994376| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.10178170576334064|0.14219939028218284| -0.3492867754665158|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313| 1|[-0.2045971821636...|[-0.2045971821636...| 17.28247725096573| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.01947813616931213|0.14219939028218284| -0.2871819088527123| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313| 1|[0.71909117704347...|[0.71909117704347...| 18.70006531503304| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.01947813616931213|0.06939409923243636|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 0|[-1.1282855413707...|[-1.1282855413707...|18.188117222794812| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.06282543342471639|0.06939409923243636|-0.24060325889236406|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 1|[-0.2045971821636...|[-0.2045971821636...|18.188117222794812| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| -0.1451290030187449|0.14219939028218284|-0.20955082558545346|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 8|[0.71909117704347...|[0.71909117704347...|19.520817025119957| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641|-0.06282543342471639| 0.2150046813319293|-0.19402460893199816|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313| 16|[-0.2045971821636...|[-0.2045971821636...| 17.28247725096573| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.10178170576334064| 0.2150046813319293|-0.14744595897164992|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 52|[-0.2045971821636...|[-0.2045971821636...| 17.28247725096573| |-1.1282855413707529|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.3486924145454262|0.36061526343142225|-0.13191974231819462|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 69|[-1.1282855413707...|[-1.1282855413707...|20.111591250048935| | 1.9506756559862992|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.5132995537334832|0.28780997238167577|-0.08534109235784636|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313| 54|[1.95067565598629...|[1.95067565598629...|22.180979483401217| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 0.6779066929215403|0.36061526343142225|-0.10086730901130166|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313| 21|[0.41119505730777...|[0.41119505730777...|20.111591250048935| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.28780997238167577|-0.16297217562510521|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313| 15|[0.41119505730777...|[0.41119505730777...|20.111591250048935| | 1.3348834165148888|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.089424540891683| 0.2150046813319293| -0.1784983922785605|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313| 15|[1.33488341651488...|[1.33488341651488...|23.780114059765907| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684| 0.2150046813319293|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313| 17|[-0.2045971821636...|[-0.2045971821636...|21.326904911897515| | 0.4111950573077732|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.25403168007974| 0.2150046813319293| -0.3337605588130782|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313| 12|[0.41119505730777...|[0.41119505730777...|22.693524979302715| |-0.2045971821636372|-0.3045476160907193| 0| 1| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684|0.14219939028218284| -0.3182343421596229| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313| 27|[-0.2045971821636...|[-0.2045971821636...|26.835392583369064| |-0.2045971821636372|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.3363352496737684|0.06939409923243636|-0.39586542542688175| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313| 47|[-0.2045971821636...|[-0.2045971821636...|21.326904911897515| | 1.0269872967791835|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.25403168007974|0.06939409923243636|-0.44244407538722996|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 165|[1.02698729677918...|[1.02698729677918...|22.484917237808524| | 0.7190911770434785|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.28780997238167577|-0.47349650869414056|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 128|[0.71909117704347...|[0.71909117704347...|22.484917237808524| | 1.0269872967791835|-0.3045476160907193| 1| 0| 0| 0| 0| 0|0.4638614904960641| 1.1717281104857113|0.06939409923243636| -0.4579702920406853|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313| 36|[1.02698729677918...|[1.02698729677918...|22.484917237808524| +-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+ only showing top 20 rows
df_rf = july_prediction3_df.select("*").toPandas()
df_rf.to_csv('df_rf.csv')
df_rf
| SPD | GUS | CLR | SCT | BKN | OVC | OBS | POB | VSB | TEMP | DEWP | SLP | ALT | STP | PCP01 | count | va_features | features | prediction | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.184085 | 0.069394 | -0.411392 | -0.400542 | -0.402984 | -0.178119 | 2 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 17.957007 |
| 1 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.101782 | 0.142199 | -0.349287 | -0.347805 | -0.340400 | -0.178119 | 1 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 17.282477 |
| 2 | 0.719091 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.019478 | 0.142199 | -0.287182 | -0.295068 | -0.293462 | -0.178119 | 1 | [0.7190911770434785, -0.3045476160907193, 1.0,... | [0.7190911770434785, -0.3045476160907193, 1.0,... | 18.700065 |
| 3 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 0.019478 | 0.069394 | -0.256129 | -0.242331 | -0.246525 | -0.178119 | 0 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 18.188117 |
| 4 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | -0.062825 | 0.069394 | -0.240603 | -0.242331 | -0.246525 | -0.178119 | 1 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 18.188117 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 727 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.097945 | 0.924817 | 1.161473 | 0.302814 | 0.337777 | 0.332374 | -0.178119 | 61 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 19.637665 |
| 728 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | -0.349287 | 1.089425 | 1.161473 | 0.333867 | 0.337777 | 0.332374 | -0.178119 | 103 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 20.346083 |
| 729 | -1.128286 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | -0.349287 | 1.171728 | 1.161473 | 0.333867 | 0.337777 | 0.332374 | -0.178119 | 64 | [-1.1282855413707529, -0.3045476160907193, 1.0... | [-1.1282855413707529, -0.3045476160907193, 1.0... | 20.346083 |
| 730 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | -0.349287 | 1.418639 | 1.161473 | 0.380445 | 0.390514 | 0.394957 | -0.178119 | 20 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 21.125246 |
| 731 | -0.204597 | -0.304548 | 1 | 0 | 0 | 0 | 0 | 0 | 0.463861 | 1.418639 | 1.088668 | 0.364919 | 0.390514 | 0.394957 | -0.178119 | 12 | [-0.2045971821636372, -0.3045476160907193, 1.0... | [-0.2045971821636372, -0.3045476160907193, 1.0... | 22.378931 |
732 rows × 19 columns
%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import geopandas as gpd
import folium
import datetime
from folium.plugins import HeatMapWithTime
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from IPython.display import HTML
import warnings
warnings.filterwarnings('ignore')
# load data and preprocess datetime
citibike = pd.read_csv('201907-citibike-tripdata.csv')
citibike['starttime'] = citibike['starttime'].str[:-5]
citibike['stoptime'] = citibike['stoptime'].str[:-5]
citibike['starttime'] = pd.to_datetime(citibike['starttime'])
citibike['stoptime'] = pd.to_datetime(citibike['stoptime'])
citibike = citibike.set_index('starttime')
# create list to store lists of locations day by day, hour by hour
df_hour_list = []
day_list = list(set(citibike.index.date))
day_list.sort()
time_index = []
for day in day_list:
for hour in range(0,24):
time_index.append(datetime.datetime.combine(day, datetime.time(hour)))
for day in day_list:
for hour in range(0,24):
citibike_day = citibike.loc[citibike.index.date == day, ['start station latitude', 'start station longitude']]
citibike_hour = citibike_day.loc[citibike_day.index.hour == hour, ['start station latitude', 'start station longitude']].groupby(['start station latitude', 'start station longitude']).sum().reset_index().values.tolist()
citibike_demand = citibike_day.loc[citibike_day.index.hour == hour, ['start station latitude', 'start station longitude']].groupby(['start station latitude', 'start station longitude']).size()
demand_max = citibike_demand.values.max()
demand_scaled = citibike_demand.values/demand_max
for k in range(len(citibike_hour)):
citibike_hour[k].append(demand_scaled[k])
df_hour_list.append(citibike_hour)
# add trip events to the map
time_index = [str(x) for x in time_index]
map_time = folium.Map(location=[40.7470, -73.9955], tiles='CartoDB Positron', zoom_start=13)
HeatMapWithTime(df_hour_list, index=time_index, auto_play=True, max_opacity=0.5, gradient={0.2: 'cornflowerblue', 0.4: 'royalblue', 0.75: 'mediumblue', 1: 'blue'}).add_to(map_time)
map_time